Index: kernel/generic/include/atomic.h =================================================================== --- kernel/generic/include/atomic.h (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/include/atomic.h (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -63,4 +63,95 @@ (new_val), memory_order_relaxed) +#if __64_BITS__ + +typedef struct { + atomic_uint_fast64_t value; +} atomic_time_stat_t; + +#define ATOMIC_TIME_INITIALIZER() (atomic_time_stat_t) {} + +static inline void atomic_time_increment(atomic_time_stat_t *time, int a) +{ + /* + * We require increments to be synchronized with each other, so we + * can use ordinary reads and writes instead of a more expensive atomic + * read-modify-write operations. + */ + uint64_t v = atomic_load_explicit(&time->value, memory_order_relaxed); + atomic_store_explicit(&time->value, v + a, memory_order_relaxed); +} + +static inline uint64_t atomic_time_read(atomic_time_stat_t *time) +{ + return atomic_load_explicit(&time->value, memory_order_relaxed); +} + +#else + +/** + * A monotonically increasing 64b time statistic. + * Increments must be synchronized with each other (or limited to a single + * thread/CPU), but reads can be performed from any thread. + * + */ +typedef struct { + uint64_t true_value; + atomic_uint_fast32_t high1; + atomic_uint_fast32_t high2; + atomic_uint_fast32_t low; +} atomic_time_stat_t; + +#define ATOMIC_TIME_INITIALIZER() (atomic_time_stat_t) {} + +static inline void atomic_time_increment(atomic_time_stat_t *time, int a) +{ + /* + * On 32b architectures, we can't rely on 64b memory reads/writes being + * architecturally atomic, but we also don't want to pay the cost of + * emulating atomic reads/writes, so instead we split value in half + * and perform some ordering magic to make sure readers always get + * consistent value. + */ + + /* true_value is only used by the writer, so this need not be atomic. */ + uint64_t val = time->true_value; + uint32_t old_high = val >> 32; + val += a; + uint32_t new_high = val >> 32; + time->true_value = val; + + /* Tell GCC that the first branch is far more likely than the second. */ + if (__builtin_expect(old_high == new_high, 1)) { + /* If the high half didn't change, we need not bother with barriers. */ + atomic_store_explicit(&time->low, (uint32_t) val, memory_order_relaxed); + } else { + /* + * If both halves changed, extra ordering is necessary. + * The idea is that if reader reads high1 and high2 with the same value, + * it is guaranteed that they read the correct low half for that value. + * + * This is the same sequence that is used by userspace to read clock. + */ + atomic_store_explicit(&time->high1, new_high, memory_order_relaxed); + atomic_store_explicit(&time->low, (uint32_t) val, memory_order_release); + atomic_store_explicit(&time->high2, new_high, memory_order_release); + } +} + +static inline uint64_t atomic_time_read(atomic_time_stat_t *time) +{ + uint32_t high2 = atomic_load_explicit(&time->high2, memory_order_acquire); + uint32_t low = atomic_load_explicit(&time->low, memory_order_acquire); + uint32_t high1 = atomic_load_explicit(&time->high1, memory_order_relaxed); + + if (high1 != high2) + low = 0; + + /* If the values differ, high1 is always the newer value. */ + return (uint64_t) high1 << 32 | (uint64_t) low; +} + +#endif /* __64_BITS__ */ + #endif Index: kernel/generic/include/cpu.h =================================================================== --- kernel/generic/include/cpu.h (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/include/cpu.h (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -81,6 +81,6 @@ bool idle; uint64_t last_cycle; - uint64_t idle_cycles; - uint64_t busy_cycles; + atomic_time_stat_t idle_cycles; + atomic_time_stat_t busy_cycles; /** Index: kernel/generic/src/cpu/cpu.c =================================================================== --- kernel/generic/src/cpu/cpu.c (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/src/cpu/cpu.c (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -103,6 +103,6 @@ CPU->idle = false; CPU->last_cycle = get_cycle(); - CPU->idle_cycles = 0; - CPU->busy_cycles = 0; + CPU->idle_cycles = ATOMIC_TIME_INITIALIZER(); + CPU->busy_cycles = ATOMIC_TIME_INITIALIZER(); cpu_identify(); Index: kernel/generic/src/interrupt/interrupt.c =================================================================== --- kernel/generic/src/interrupt/interrupt.c (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/src/interrupt/interrupt.c (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -116,5 +116,5 @@ irq_spinlock_lock(&CPU->lock, false); uint64_t now = get_cycle(); - CPU->idle_cycles += now - CPU->last_cycle; + atomic_time_increment(&CPU->idle_cycles, now - CPU->last_cycle); CPU->last_cycle = now; CPU->idle = false; Index: kernel/generic/src/sysinfo/stats.c =================================================================== --- kernel/generic/src/sysinfo/stats.c (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/src/sysinfo/stats.c (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -124,6 +124,7 @@ stats_cpus[i].active = cpus[i].active; stats_cpus[i].frequency_mhz = cpus[i].frequency_mhz; - stats_cpus[i].busy_cycles = cpus[i].busy_cycles; - stats_cpus[i].idle_cycles = cpus[i].idle_cycles; + + stats_cpus[i].busy_cycles = atomic_time_read(&cpus[i].busy_cycles); + stats_cpus[i].idle_cycles = atomic_time_read(&cpus[i].idle_cycles); irq_spinlock_unlock(&cpus[i].lock, true); Index: kernel/generic/src/time/clock.c =================================================================== --- kernel/generic/src/time/clock.c (revision dd218ea076a20828a183015319cffbd31d1769de) +++ kernel/generic/src/time/clock.c (revision b2ec5cfd5e00eac7963f5a84fb71f002ad79f1f0) @@ -125,5 +125,5 @@ irq_spinlock_lock(&CPU->lock, false); uint64_t now = get_cycle(); - CPU->busy_cycles += now - CPU->last_cycle; + atomic_time_increment(&CPU->busy_cycles, now - CPU->last_cycle); CPU->last_cycle = now; irq_spinlock_unlock(&CPU->lock, false);