Index: HelenOS.config =================================================================== --- HelenOS.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ HelenOS.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -385,4 +385,9 @@ ! [COMPILER=gcc_cross|COMPILER=gcc_native] CONFIG_LTO (n/y) +% Kernel RCU algorithm +@ "PREEMPT_PODZIMEK" Preemptible Podzimek-RCU +@ "PREEMPT_A" Preemptible A-RCU +! RCU (choice) + ## Hardware support Index: defaults/amd64/Makefile.config =================================================================== --- defaults/amd64/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/amd64/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -38,4 +38,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/arm32/Makefile.config =================================================================== --- defaults/arm32/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/arm32/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -29,4 +29,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # What is your input device? CONFIG_HID_IN = generic Index: defaults/ia32/Makefile.config =================================================================== --- defaults/ia32/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/ia32/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -44,4 +44,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/ia64/Makefile.config =================================================================== --- defaults/ia64/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/ia64/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/mips32/Makefile.config =================================================================== --- defaults/mips32/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/mips32/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -35,4 +35,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/mips64/Makefile.config =================================================================== --- defaults/mips64/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/mips64/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -35,4 +35,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/ppc32/Makefile.config =================================================================== --- defaults/ppc32/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/ppc32/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -29,4 +29,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: defaults/sparc64/Makefile.config =================================================================== --- defaults/sparc64/Makefile.config (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ defaults/sparc64/Makefile.config (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -44,4 +44,7 @@ CONFIG_TEST = y +# Kernel RCU implementation +RCU = PREEMPT_A + # Input device class CONFIG_HID_IN = generic Index: kernel/Makefile =================================================================== --- kernel/Makefile (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/Makefile (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -192,4 +192,5 @@ generic/src/adt/bitmap.c \ generic/src/adt/btree.c \ + generic/src/adt/cht.c \ generic/src/adt/hash_table.c \ generic/src/adt/list.c \ @@ -198,4 +199,5 @@ generic/src/console/prompt.c \ generic/src/cpu/cpu.c \ + generic/src/cpu/cpu_mask.c \ generic/src/ddi/ddi.c \ generic/src/ddi/irq.c \ @@ -253,6 +255,9 @@ generic/src/synch/waitq.c \ generic/src/synch/futex.c \ + generic/src/synch/workqueue.c \ + generic/src/synch/rcu.c \ generic/src/smp/ipi.c \ generic/src/smp/smp.c \ + generic/src/smp/smp_call.c \ generic/src/ipc/ipc.c \ generic/src/ipc/sysipc.c \ @@ -293,4 +298,5 @@ test/atomic/atomic1.c \ test/btree/btree1.c \ + test/cht/cht1.c \ test/avltree/avltree1.c \ test/fault/fault1.c \ @@ -302,4 +308,7 @@ test/synch/semaphore1.c \ test/synch/semaphore2.c \ + test/synch/workqueue2.c \ + test/synch/workqueue3.c \ + test/synch/rcu1.c \ test/print/print1.c \ test/print/print2.c \ @@ -307,5 +316,6 @@ test/print/print4.c \ test/print/print5.c \ - test/thread/thread1.c + test/thread/thread1.c \ + test/smpcall/smpcall1.c ifeq ($(KARCH),mips32) Index: kernel/arch/amd64/Makefile.inc =================================================================== --- kernel/arch/amd64/Makefile.inc (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/Makefile.inc (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -104,4 +104,5 @@ arch/$(KARCH)/src/smp/ipi.c \ arch/$(KARCH)/src/smp/mps.c \ + arch/$(KARCH)/src/smp/smp_call.c \ arch/$(KARCH)/src/smp/smp.c endif Index: kernel/arch/amd64/include/atomic.h =================================================================== --- kernel/arch/amd64/include/atomic.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/include/atomic.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -140,4 +140,114 @@ } + +#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \ + asm volatile ( \ + prefix " cmpxchgq %[newval], %[ptr]\n" \ + : /* Output operands. */ \ + /* Old/current value is returned in eax. */ \ + [oldval] "=a" (old_val), \ + /* (*ptr) will be read and written to, hence "+" */ \ + [ptr] "+m" (*pptr) \ + : /* Input operands. */ \ + /* Expected value must be in eax. */ \ + [expval] "a" (exp_val), \ + /* The new value may be in any register. */ \ + [newval] "r" (new_val) \ + : "memory" \ + ) + +/** Atomically compares and swaps the pointer at pptr. */ +NO_TRACE static inline void * atomic_cas_ptr(void **pptr, + void *exp_val, void *new_val) +{ + void *old_val; + _atomic_cas_impl(pptr, exp_val, new_val, old_val, "lock\n"); + return old_val; +} + +/** Compare-and-swap of a pointer that is atomic wrt to local cpu's interrupts. + * + * This function is NOT smp safe and is not atomic with respect to other cpus. + */ +NO_TRACE static inline void * atomic_cas_ptr_local(void **pptr, + void *exp_val, void *new_val) +{ + void *old_val; + _atomic_cas_impl(pptr, exp_val, new_val, old_val, ""); + return old_val; +} + + +#define _atomic_swap_impl(pptr, new_val) \ +({ \ + typeof(*(pptr)) new_in_old_out = new_val; \ + asm volatile ( \ + "xchgq %[val], %[p_ptr]\n" \ + : [val] "+r" (new_in_old_out), \ + [p_ptr] "+m" (*pptr) \ + ); \ + \ + new_in_old_out; \ +}) + +/* + * Issuing a xchg instruction always implies lock prefix semantics. + * Therefore, it is cheaper to use a cmpxchg without a lock prefix + * in a loop. + */ +#define _atomic_swap_local_impl(pptr, new_val) \ +({ \ + typeof(*(pptr)) exp_val; \ + typeof(*(pptr)) old_val; \ + \ + do { \ + exp_val = *pptr; \ + _atomic_cas_impl(pptr, exp_val, new_val, old_val, ""); \ + } while (old_val != exp_val); \ + \ + old_val; \ +}) + + +/** Atomicaly sets *ptr to val and returns the previous value. */ +NO_TRACE static inline void * atomic_set_return_ptr(void **pptr, void *val) +{ + return _atomic_swap_impl(pptr, val); +} + +/** Sets *ptr to new_val and returns the previous value. NOT smp safe. + * + * This function is only atomic wrt to local interrupts and it is + * NOT atomic wrt to other cpus. + */ +NO_TRACE static inline void * atomic_set_return_ptr_local( + void **pptr, void *new_val) +{ + return _atomic_swap_local_impl(pptr, new_val); +} + +/** Atomicaly sets *ptr to val and returns the previous value. */ +NO_TRACE static inline native_t atomic_set_return_native_t( + native_t *p, native_t val) +{ + return _atomic_swap_impl(p, val); +} + +/** Sets *ptr to new_val and returns the previous value. NOT smp safe. + * + * This function is only atomic wrt to local interrupts and it is + * NOT atomic wrt to other cpus. + */ +NO_TRACE static inline native_t atomic_set_return_native_t_local( + native_t *p, native_t new_val) +{ + return _atomic_swap_local_impl(p, new_val); +} + + +#undef _atomic_cas_ptr_impl +#undef _atomic_swap_impl +#undef _atomic_swap_local_impl + #endif Index: kernel/arch/amd64/include/cpu.h =================================================================== --- kernel/arch/amd64/include/cpu.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/include/cpu.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -73,4 +73,6 @@ tss_t *tss; + unsigned int id; /** CPU's local, ie physical, APIC ID. */ + size_t iomapver_copy; /** Copy of TASK's I/O Permission bitmap generation count. */ } cpu_arch_t; Index: kernel/arch/amd64/include/interrupt.h =================================================================== --- kernel/arch/amd64/include/interrupt.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/include/interrupt.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -69,4 +69,5 @@ #define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) #define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) +#define VECTOR_SMP_CALL_IPI (IVT_FREEBASE + 3) extern void (* disable_irqs_function)(uint16_t); Index: kernel/arch/amd64/src/amd64.c =================================================================== --- kernel/arch/amd64/src/amd64.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/src/amd64.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -171,5 +171,5 @@ } -void arch_post_cpu_init() +void arch_post_cpu_init(void) { #ifdef CONFIG_SMP Index: kernel/arch/amd64/src/cpu/cpu.c =================================================================== --- kernel/arch/amd64/src/cpu/cpu.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/src/cpu/cpu.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -158,7 +158,7 @@ void cpu_print_report(cpu_t* m) { - printf("cpu%d: (%s family=%d model=%d stepping=%d) %dMHz\n", + printf("cpu%d: (%s family=%d model=%d stepping=%d apicid=%u) %dMHz\n", m->id, vendor_str[m->arch.vendor], m->arch.family, m->arch.model, - m->arch.stepping, m->frequency_mhz); + m->arch.stepping, m->arch.id, m->frequency_mhz); } Index: kernel/arch/amd64/src/interrupt.c =================================================================== --- kernel/arch/amd64/src/interrupt.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/amd64/src/interrupt.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -54,4 +54,5 @@ #include #include +#include /* @@ -161,4 +162,10 @@ tlb_shootdown_ipi_recv(); } + +static void arch_smp_call_ipi_recv(unsigned int n, istate_t *istate) +{ + trap_virtual_eoi(); + smp_call_ipi_recv(); +} #endif @@ -222,4 +229,6 @@ exc_register(VECTOR_TLB_SHOOTDOWN_IPI, "tlb_shootdown", true, (iroutine_t) tlb_shootdown_ipi); + exc_register(VECTOR_SMP_CALL_IPI, "smp_call", true, + (iroutine_t) arch_smp_call_ipi_recv); #endif } Index: kernel/arch/amd64/src/smp/smp_call.c =================================================================== --- kernel/arch/amd64/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/arch/amd64/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,1 @@ +../../../ia32/src/smp/smp_call.c Index: kernel/arch/ia32/Makefile.inc =================================================================== --- kernel/arch/ia32/Makefile.inc (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/Makefile.inc (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -86,4 +86,5 @@ arch/$(KARCH)/src/smp/mps.c \ arch/$(KARCH)/src/smp/smp.c \ + arch/$(KARCH)/src/smp/smp_call.c \ arch/$(KARCH)/src/atomic.S \ arch/$(KARCH)/src/smp/ipi.c \ Index: kernel/arch/ia32/include/atomic.h =================================================================== --- kernel/arch/ia32/include/atomic.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/include/atomic.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -113,4 +113,5 @@ } + /** ia32 specific fast spinlock */ NO_TRACE static inline void atomic_lock_arch(atomic_t *val) @@ -142,4 +143,114 @@ } + +#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \ + asm volatile ( \ + prefix " cmpxchgl %[newval], %[ptr]\n" \ + : /* Output operands. */ \ + /* Old/current value is returned in eax. */ \ + [oldval] "=a" (old_val), \ + /* (*ptr) will be read and written to, hence "+" */ \ + [ptr] "+m" (*pptr) \ + : /* Input operands. */ \ + /* Expected value must be in eax. */ \ + [expval] "a" (exp_val), \ + /* The new value may be in any register. */ \ + [newval] "r" (new_val) \ + : "memory" \ + ) + +/** Atomically compares and swaps the pointer at pptr. */ +NO_TRACE static inline void * atomic_cas_ptr(void **pptr, + void *exp_val, void *new_val) +{ + void *old_val; + _atomic_cas_impl(pptr, exp_val, new_val, old_val, "lock\n"); + return old_val; +} + +/** Compare-and-swap of a pointer that is atomic wrt to local cpu's interrupts. + * + * This function is NOT smp safe and is not atomic with respect to other cpus. + */ +NO_TRACE static inline void * atomic_cas_ptr_local(void **pptr, + void *exp_val, void *new_val) +{ + void *old_val; + _atomic_cas_impl(pptr, exp_val, new_val, old_val, ""); + return old_val; +} + + +#define _atomic_swap_impl(pptr, new_val) \ +({ \ + typeof(*(pptr)) new_in_old_out = new_val; \ + asm volatile ( \ + "xchgl %[val], %[p_ptr]\n" \ + : [val] "+r" (new_in_old_out), \ + [p_ptr] "+m" (*pptr) \ + ); \ + \ + new_in_old_out; \ +}) + +/* + * Issuing a xchg instruction always implies lock prefix semantics. + * Therefore, it is cheaper to use a cmpxchg without a lock prefix + * in a loop. + */ +#define _atomic_swap_local_impl(pptr, new_val) \ +({ \ + typeof(*(pptr)) exp_val; \ + typeof(*(pptr)) old_val; \ + \ + do { \ + exp_val = *pptr; \ + _atomic_cas_impl(pptr, exp_val, new_val, old_val, ""); \ + } while (old_val != exp_val); \ + \ + old_val; \ +}) + + +/** Atomicaly sets *ptr to val and returns the previous value. */ +NO_TRACE static inline void * atomic_set_return_ptr(void **pptr, void *val) +{ + return _atomic_swap_impl(pptr, val); +} + +/** Sets *ptr to new_val and returns the previous value. NOT smp safe. + * + * This function is only atomic wrt to local interrupts and it is + * NOT atomic wrt to other cpus. + */ +NO_TRACE static inline void * atomic_set_return_ptr_local( + void **pptr, void *new_val) +{ + return _atomic_swap_local_impl(pptr, new_val); +} + +/** Atomicaly sets *ptr to val and returns the previous value. */ +NO_TRACE static inline native_t atomic_set_return_native_t( + native_t *p, native_t val) +{ + return _atomic_swap_impl(p, val); +} + +/** Sets *ptr to new_val and returns the previous value. NOT smp safe. + * + * This function is only atomic wrt to local interrupts and it is + * NOT atomic wrt to other cpus. + */ +NO_TRACE static inline native_t atomic_set_return_native_t_local( + native_t *p, native_t new_val) +{ + return _atomic_swap_local_impl(p, new_val); +} + + +#undef _atomic_cas_ptr_impl +#undef _atomic_swap_impl +#undef _atomic_swap_local_impl + #endif Index: kernel/arch/ia32/include/cpu.h =================================================================== --- kernel/arch/ia32/include/cpu.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/include/cpu.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -60,4 +60,6 @@ unsigned int stepping; cpuid_feature_info fi; + + unsigned int id; /** CPU's local, ie physical, APIC ID. */ tss_t *tss; Index: kernel/arch/ia32/include/interrupt.h =================================================================== --- kernel/arch/ia32/include/interrupt.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/include/interrupt.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -69,4 +69,5 @@ #define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) #define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) +#define VECTOR_SMP_CALL_IPI (IVT_FREEBASE + 3) extern void (* disable_irqs_function)(uint16_t); Index: kernel/arch/ia32/include/smp/apic.h =================================================================== --- kernel/arch/ia32/include/smp/apic.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/include/smp/apic.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -353,4 +353,5 @@ extern void l_apic_init(void); extern void l_apic_eoi(void); +extern int l_apic_send_custom_ipi(uint8_t, uint8_t); extern int l_apic_broadcast_custom_ipi(uint8_t); extern int l_apic_send_init_ipi(uint8_t); Index: kernel/arch/ia32/src/cpu/cpu.c =================================================================== --- kernel/arch/ia32/src/cpu/cpu.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/src/cpu/cpu.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -160,7 +160,7 @@ void cpu_print_report(cpu_t* cpu) { - printf("cpu%u: (%s family=%u model=%u stepping=%u) %" PRIu16 " MHz\n", - cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family, - cpu->arch.model, cpu->arch.stepping, cpu->frequency_mhz); + printf("cpu%u: (%s family=%u model=%u stepping=%u apicid=%u) %" PRIu16 + " MHz\n", cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family, + cpu->arch.model, cpu->arch.stepping, cpu->arch.id, cpu->frequency_mhz); } Index: kernel/arch/ia32/src/ia32.c =================================================================== --- kernel/arch/ia32/src/ia32.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/src/ia32.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -125,5 +125,5 @@ } -void arch_post_cpu_init() +void arch_post_cpu_init(void) { #ifdef CONFIG_SMP Index: kernel/arch/ia32/src/interrupt.c =================================================================== --- kernel/arch/ia32/src/interrupt.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/src/interrupt.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -54,4 +54,6 @@ #include #include +#include +#include /* @@ -170,4 +172,10 @@ tlb_shootdown_ipi_recv(); } + +static void arch_smp_call_ipi_recv(unsigned int n, istate_t *istate) +{ + trap_virtual_eoi(); + smp_call_ipi_recv(); +} #endif @@ -230,4 +238,6 @@ exc_register(VECTOR_TLB_SHOOTDOWN_IPI, "tlb_shootdown", true, (iroutine_t) tlb_shootdown_ipi); + exc_register(VECTOR_SMP_CALL_IPI, "smp_call", true, + (iroutine_t) arch_smp_call_ipi_recv); #endif } Index: kernel/arch/ia32/src/smp/apic.c =================================================================== --- kernel/arch/ia32/src/smp/apic.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/src/smp/apic.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -259,22 +259,44 @@ } -#define DELIVS_PENDING_SILENT_RETRIES 4 - +/* Waits for the destination cpu to accept the previous ipi. */ static void l_apic_wait_for_delivery(void) { icr_t icr; - unsigned retries = 0; - + do { - if (retries++ > DELIVS_PENDING_SILENT_RETRIES) { - retries = 0; -#ifdef CONFIG_DEBUG - printf("IPI is pending.\n"); -#endif - delay(20); - } icr.lo = l_apic[ICRlo]; - } while (icr.delivs == DELIVS_PENDING); - + } while (icr.delivs != DELIVS_IDLE); +} + +/** Send one CPU an IPI vector. + * + * @param apicid Physical APIC ID of the destination CPU. + * @param vector Interrupt vector to be sent. + * + * @return 0 on failure, 1 on success. + */ +int l_apic_send_custom_ipi(uint8_t apicid, uint8_t vector) +{ + icr_t icr; + + /* Wait for a destination cpu to accept our previous ipi. */ + l_apic_wait_for_delivery(); + + icr.lo = l_apic[ICRlo]; + icr.hi = l_apic[ICRhi]; + + icr.delmod = DELMOD_FIXED; + icr.destmod = DESTMOD_PHYS; + icr.level = LEVEL_ASSERT; + icr.shorthand = SHORTHAND_NONE; + icr.trigger_mode = TRIGMOD_LEVEL; + icr.vector = vector; + icr.dest = apicid; + + /* Send the IPI by writing to l_apic[ICRlo]. */ + l_apic[ICRhi] = icr.hi; + l_apic[ICRlo] = icr.lo; + + return apic_poll_errors(); } @@ -289,4 +311,7 @@ { icr_t icr; + + /* Wait for a destination cpu to accept our previous ipi. */ + l_apic_wait_for_delivery(); icr.lo = l_apic[ICRlo]; @@ -299,6 +324,4 @@ l_apic[ICRlo] = icr.lo; - - l_apic_wait_for_delivery(); return apic_poll_errors(); Index: kernel/arch/ia32/src/smp/smp.c =================================================================== --- kernel/arch/ia32/src/smp/smp.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/arch/ia32/src/smp/smp.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -55,4 +55,5 @@ #include #include +#include #ifdef CONFIG_SMP @@ -77,4 +78,14 @@ io_apic = (uint32_t *) km_map((uintptr_t) io_apic, PAGE_SIZE, PAGE_WRITE | PAGE_NOT_CACHEABLE); + } +} + +static void cpu_arch_id_init(void) +{ + ASSERT(ops != NULL); + ASSERT(cpus != NULL); + + for (unsigned int i = 0; i < config.cpu_count; ++i) { + cpus[i].arch.id = ops->cpu_apic_id(i); } } @@ -92,4 +103,10 @@ ASSERT(ops != NULL); + + /* + * SMP initialized, cpus array allocated. Assign each CPU its + * physical APIC ID. + */ + cpu_arch_id_init(); /* Index: kernel/arch/ia32/src/smp/smp_call.c =================================================================== --- kernel/arch/ia32/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/arch/ia32/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,14 @@ +#include +#include +#include +#include + +#ifdef CONFIG_SMP + +void arch_smp_call_ipi(unsigned int cpu_id) +{ + (void) l_apic_send_custom_ipi(cpus[cpu_id].arch.id, VECTOR_SMP_CALL_IPI); +} + +#endif /* CONFIG_SMP */ + Index: kernel/generic/include/adt/cht.h =================================================================== --- kernel/generic/include/adt/cht.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/adt/cht.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup genericadt + * @{ + */ +/** @file + */ + +#ifndef KERN_CONC_HASH_TABLE_H_ +#define KERN_CONC_HASH_TABLE_H_ + +#include +#include +#include +#include +#include + +typedef uintptr_t cht_ptr_t; + +/** Concurrent hash table node link. */ +typedef struct cht_link { + /* Must be placed first. + * + * The function pointer (rcu_link.func) is used to store the item's + * memoized hash. + */ + union { + rcu_item_t rcu_link; + size_t hash; + }; + /** Link to the next item in the bucket including any marks. */ + cht_ptr_t link; +} cht_link_t; + +/** Set of operations for a concurrent hash table. */ +typedef struct cht_ops { + size_t (*hash)(const cht_link_t *item); + size_t (*key_hash)(void *key); + bool (*equal)(const cht_link_t *item1, const cht_link_t *item2); + bool (*key_equal)(void *key, const cht_link_t *item); + void (*remove_callback)(cht_link_t *item); +} cht_ops_t; + + +typedef struct cht_buckets { + size_t order; + cht_ptr_t head[1]; +} cht_buckets_t; + +/** Concurrent hash table structure. */ +typedef struct { + cht_ops_t *op; + + cht_buckets_t *b; + cht_buckets_t *new_b; + size_t invalid_hash; + + size_t min_order; + size_t max_load; + work_t resize_work; + atomic_t resize_reqs; + + atomic_t item_cnt; +} cht_t; + +#define cht_get_inst(item, type, member) \ + member_to_inst((item), type, member) + + +#define cht_read_lock() rcu_read_lock() +#define cht_read_unlock() rcu_read_unlock() + +extern bool cht_create(cht_t *h, size_t init_size, size_t min_size, + size_t max_load, cht_ops_t *op); +extern void cht_destroy(cht_t *h); + +extern cht_link_t *cht_find(cht_t *h, void *key); +extern cht_link_t *cht_find_lazy(cht_t *h, void *key); +extern cht_link_t *cht_find_next(cht_t *h, const cht_link_t *item); +extern cht_link_t *cht_find_next_lazy(cht_t *h, const cht_link_t *item); + +extern void cht_insert(cht_t *h, cht_link_t *item); +extern bool cht_insert_unique(cht_t *h, cht_link_t *item); +extern size_t cht_remove_key(cht_t *h, void *key); +extern bool cht_remove_item(cht_t *h, cht_link_t *item); + +#endif + +/** @} + */ Index: kernel/generic/include/adt/hash.h =================================================================== --- kernel/generic/include/adt/hash.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/adt/hash.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup genericadt + * @{ + */ +/** @file + */ +#ifndef KERN_HASH_H_ +#define KERN_HASH_H_ + +#include + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline uint32_t hash_mix32(uint32_t hash) +{ + /* + * Thomas Wang's modification of Bob Jenkin's hash mixing function: + * http://www.concentric.net/~Ttwang/tech/inthash.htm + * Public domain. + */ + hash = ~hash + (hash << 15); + hash = hash ^ (hash >> 12); + hash = hash + (hash << 2); + hash = hash ^ (hash >> 4); + hash = hash * 2057; + hash = hash ^ (hash >> 16); + return hash; +} + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline uint64_t hash_mix64(uint64_t hash) +{ + /* + * Thomas Wang's public domain 64-bit hash mixing function: + * http://www.concentric.net/~Ttwang/tech/inthash.htm + */ + hash = (hash ^ 61) ^ (hash >> 16); + hash = hash + (hash << 3); + hash = hash ^ (hash >> 4); + hash = hash * 0x27d4eb2d; + hash = hash ^ (hash >> 15); + /* + * Lower order bits are mixed more thoroughly. Swap them with + * the higher order bits and make the resulting higher order bits + * more usable. + */ + return (hash << 32) | (hash >> 32); +} + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline size_t hash_mix(size_t hash) +{ +#ifdef __32_BITS__ + return hash_mix32(hash); +#elif defined(__64_BITS__) + return hash_mix64(hash); +#else +#error Unknown size_t size - cannot select proper hash mix function. +#endif +} + +/** Use to create a hash from multiple values. + * + * Typical usage: + * @code + * int car_id; + * bool car_convertible; + * // .. + * size_t hash = 0; + * hash = hash_combine(hash, car_id); + * hash = hash_combine(hash, car_convertible); + * // Now use hash as a hash of both car_id and car_convertible. + * @endcode + */ +static inline size_t hash_combine(size_t seed, size_t hash) +{ + /* + * todo: use Bob Jenkin's proper mixing hash pass: + * http://burtleburtle.net/bob/c/lookup3.c + */ + seed ^= hash + 0x9e3779b9 + + ((seed << 5) | (seed >> (sizeof(size_t) * 8 - 5))); + return seed; +} + +#endif Index: kernel/generic/include/adt/list.h =================================================================== --- kernel/generic/include/adt/list.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/adt/list.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -51,4 +51,10 @@ } list_t; + +extern int list_member(const link_t *, const list_t *); +extern void list_splice(list_t *, link_t *); +extern unsigned int list_count(const list_t *); + + /** Declare and initialize statically allocated list. * @@ -71,4 +77,36 @@ iterator != &(list).head; iterator = iterator->next) +/** Unlike list_foreach(), allows removing items while traversing a list. + * + * @code + * list_t mylist; + * typedef struct item { + * int value; + * link_t item_link; + * } item_t; + * + * //.. + * + * // Print each list element's value and remove the element from the list. + * list_foreach_safe(mylist, cur_link, next_link) { + * item_t *cur_item = list_get_instance(cur_link, item_t, item_link); + * printf("%d\n", cur_item->value); + * list_remove(cur_link); + * } + * @endcode + * + * @param list List to traverse. + * @param iterator Iterator to the current element of the list. + * The item this iterator points may be safely removed + * from the list. + * @param next_iter Iterator to the next element of the list. + */ +#define list_foreach_safe(list, iterator, next_iter) \ + for (link_t *iterator = (list).head.next, \ + *next_iter = iterator->next; \ + iterator != &(list).head; \ + iterator = next_iter, next_iter = iterator->next) + + #define assert_link_not_used(link) \ ASSERT(((link)->prev == NULL) && ((link)->next == NULL)) @@ -85,4 +123,15 @@ link->prev = NULL; link->next = NULL; +} + +/** Returns true if the initialized link is already in use by any list. + * + * @param link Link to examine whether if belongs to a list or not. + * @return 1 if the link is part of a list. + * @return 0 otherwise. + */ +NO_TRACE static inline int link_used(const link_t *link) +{ + return link->prev != NULL || link->next != NULL; } @@ -256,4 +305,19 @@ { headless_list_split_or_concat(part1, part2); +} + +/** Concatenate two lists + * + * Concatenate lists @a list1 and @a list2, producing a single + * list @a list1 containing items from both (in @a list1, @a list2 + * order) and empty list @a list2. + * + * @param list1 First list and concatenated output + * @param list2 Second list and empty output. + * + */ +NO_TRACE static inline void list_concat(list_t *list1, list_t *list2) +{ + list_splice(list2, list1->head.prev); } @@ -281,8 +345,4 @@ } -extern int list_member(const link_t *, const list_t *); -extern void list_concat(list_t *, list_t *); -extern unsigned int list_count(const list_t *); - #endif Index: kernel/generic/include/arch.h =================================================================== --- kernel/generic/include/arch.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/arch.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -36,8 +36,8 @@ #define KERN_ARCH_H_ -#include -#include -#include -#include +#include /* arch_pre_main() */ +#include /* get_stack_base() */ +#include + /* @@ -49,9 +49,4 @@ #define THE ((the_t * )(get_stack_base())) -#define CPU THE->cpu -#define THREAD THE->thread -#define TASK THE->task -#define AS THE->as -#define PREEMPTION_DISABLED THE->preemption_disabled #define MAGIC UINT32_C(0xfacefeed) @@ -62,4 +57,10 @@ ((THE->task) ? (THE->task->container) : (DEFAULT_CONTAINER)) +/* Fwd decl. to avoid include hell. */ +struct thread; +struct task; +struct cpu; +struct as; + /** * For each possible kernel stack, structure @@ -68,10 +69,13 @@ */ typedef struct { - size_t preemption_disabled; /**< Preemption disabled counter. */ - thread_t *thread; /**< Current thread. */ - task_t *task; /**< Current task. */ - cpu_t *cpu; /**< Executing cpu. */ - as_t *as; /**< Current address space. */ - uint32_t magic; /**< Magic value */ + size_t preemption; /**< Preemption disabled counter and flag. */ +#ifdef RCU_PREEMPT_A + size_t rcu_nesting; /**< RCU nesting count and flag. */ +#endif + struct thread *thread; /**< Current thread. */ + struct task *task; /**< Current task. */ + struct cpu *cpu; /**< Executing cpu. */ + struct as *as; /**< Current address space. */ + uint32_t magic; /**< Magic value */ } the_t; @@ -91,4 +95,5 @@ extern void *arch_construct_function(fncptr_t *, void *, void *); + #endif Index: kernel/generic/include/compiler/barrier.h =================================================================== --- kernel/generic/include/compiler/barrier.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/compiler/barrier.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,10 @@ + +#ifndef KERN_COMPILER_BARRIER_H_ +#define KERN_COMPILER_BARRIER_H_ + +#define compiler_barrier() asm volatile ("" ::: "memory") + +/** Forces the compiler to access (ie load/store) the variable only once. */ +#define ACCESS_ONCE(var) (*((volatile typeof(var)*)&(var))) + +#endif /* KERN_COMPILER_BARRIER_H_ */ Index: kernel/generic/include/cpu.h =================================================================== --- kernel/generic/include/cpu.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/cpu.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -38,7 +38,13 @@ #include #include +#include #include #include #include +#include +#include + +#define CPU THE->cpu + /** CPU structure. @@ -94,4 +100,13 @@ /** + * SMP calls to invoke on this CPU. + */ + SPINLOCK_DECLARE(smp_calls_lock); + list_t smp_pending_calls; + + /** RCU per-cpu data. Uses own locking. */ + rcu_cpu_data_t rcu; + + /** * Stack used by scheduler when there is no running thread. */ Index: kernel/generic/include/cpu/cpu_mask.h =================================================================== --- kernel/generic/include/cpu/cpu_mask.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/cpu/cpu_mask.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ +/** @file + */ +#ifndef KERN_CPU_CPU_MASK_H_ +#define KERN_CPU_CPU_MASK_H_ + +#include +#include +#include + +/** Iterates over all cpu id's whose bit is included in the cpu mask. + * + * Example usage: + * @code + * DEFINE_CPU_MASK(cpu_mask); + * cpu_mask_active(&cpu_mask); + * + * cpu_mask_for_each(cpu_mask, cpu_id) { + * printf("Cpu with logical id %u is active.\n", cpu_id); + * } + * @endcode + */ +#define cpu_mask_for_each(mask, cpu_id) \ + for (unsigned int (cpu_id) = 0; (cpu_id) < config.cpu_count; ++(cpu_id)) \ + if (cpu_mask_is_set(&(mask), (cpu_id))) + +/** Allocates a cpu_mask_t on stack. */ +#define DEFINE_CPU_MASK(cpu_mask) \ + cpu_mask_t *(cpu_mask) = (cpu_mask_t*) alloca(cpu_mask_size()) + +/** If used with DEFINE_CPU_MASK, the mask is large enough for all detected cpus.*/ +typedef struct cpu_mask { + unsigned int mask[1]; +} cpu_mask_t; + + +extern size_t cpu_mask_size(void); +extern void cpu_mask_active(cpu_mask_t *); +extern void cpu_mask_all(cpu_mask_t *); +extern void cpu_mask_none(cpu_mask_t *); +extern void cpu_mask_set(cpu_mask_t *, unsigned int); +extern void cpu_mask_reset(cpu_mask_t *, unsigned int); +extern bool cpu_mask_is_set(cpu_mask_t *, unsigned int); +extern bool cpu_mask_is_none(cpu_mask_t *); + +#endif /* KERN_CPU_CPU_MASK_H_ */ + +/** @} + */ Index: kernel/generic/include/lib/memfnc.h =================================================================== --- kernel/generic/include/lib/memfnc.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/lib/memfnc.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,6 @@ extern void *memcpy(void *, const void *, size_t); +#define alloca(size) __builtin_alloca((size)) + #endif Index: kernel/generic/include/macros.h =================================================================== --- kernel/generic/include/macros.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/macros.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -131,4 +131,11 @@ }) + +#ifndef member_to_inst +#define member_to_inst(ptr_member, type, member_identif) \ + ((type*) (((void*)(ptr_member)) - ((void*)&(((type*)0)->member_identif)))) +#endif + + #endif Index: kernel/generic/include/memstr.h =================================================================== --- kernel/generic/include/memstr.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/memstr.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -40,4 +40,5 @@ #define memset(dst, val, cnt) __builtin_memset((dst), (val), (cnt)) #define memcpy(dst, src, cnt) __builtin_memcpy((dst), (src), (cnt)) +#define bzero(dst, cnt) memset((dst), 0, (cnt)) extern void memsetb(void *, size_t, uint8_t); Index: kernel/generic/include/mm/as.h =================================================================== --- kernel/generic/include/mm/as.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/mm/as.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -48,4 +48,8 @@ #include #include +#include + +#define AS THE->as + /** Index: kernel/generic/include/preemption.h =================================================================== --- kernel/generic/include/preemption.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/preemption.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -36,6 +36,27 @@ #define KERN_PREEMPTION_H_ -extern void preemption_disable(void); -extern void preemption_enable(void); +#include +#include +#include + +#define PREEMPTION_INC (1 << 0) +#define PREEMPTION_DISABLED (PREEMPTION_INC <= THE->preemption) +#define PREEMPTION_ENABLED (!PREEMPTION_DISABLED) + +/** Increment preemption disabled counter. */ +#define preemption_disable() \ + do { \ + THE->preemption += PREEMPTION_INC; \ + compiler_barrier(); \ + } while (0) + +/** Restores preemption but never reschedules. */ +#define preemption_enable() \ + do { \ + ASSERT(PREEMPTION_DISABLED); \ + compiler_barrier(); \ + THE->preemption -= PREEMPTION_INC; \ + } while (0) + #endif Index: kernel/generic/include/proc/task.h =================================================================== --- kernel/generic/include/proc/task.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/proc/task.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -57,4 +57,8 @@ #include #include +#include + +#define TASK THE->task + struct thread; Index: kernel/generic/include/proc/thread.h =================================================================== --- kernel/generic/include/proc/thread.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/proc/thread.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include #include #include @@ -48,4 +49,8 @@ #include #include +#include + + +#define THREAD THE->thread #define THREAD_NAME_BUFLEN 20 @@ -180,4 +185,16 @@ /** Thread ID. */ thread_id_t tid; + + /** Work queue this thread belongs to or NULL. Immutable. */ + struct work_queue *workq; + /** Links work queue threads. Protected by workq->lock. */ + link_t workq_link; + /** True if the worker was blocked and is not running. Use thread->lock. */ + bool workq_blocked; + /** True if the worker will block in order to become idle. Use workq->lock. */ + bool workq_idling; + + /** RCU thread related data. Protected by its own locks. */ + rcu_thread_data_t rcu; /** Architecture-specific data. */ @@ -217,4 +234,6 @@ extern void thread_ready(thread_t *); extern void thread_exit(void) __attribute__((noreturn)); +extern void thread_interrupt(thread_t *); +extern bool thread_interrupted(thread_t *); #ifndef thread_create_arch Index: kernel/generic/include/smp/smp_call.h =================================================================== --- kernel/generic/include/smp/smp_call.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/smp/smp_call.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ +/** @file + */ + +#ifndef KERN_SMP_CALL_H_ +#define KERN_SMP_CALL_H_ + +#include +#include +#include + +typedef void (*smp_call_func_t)(void *); + +typedef struct smp_call { + smp_call_func_t func; + void *arg; + link_t calls_link; + atomic_t pending; +} smp_call_t; + + + +extern void smp_call(unsigned int, smp_call_func_t, void *); +extern void smp_call_async(unsigned int, smp_call_func_t, void *, smp_call_t *); +extern void smp_call_wait(smp_call_t *); + +extern void smp_call_init(void); + +#ifdef CONFIG_SMP +extern void smp_call_ipi_recv(void); +extern void arch_smp_call_ipi(unsigned int); +#endif + + + + +#endif /* KERN_SMP_CALL_H_ */ + +/** @} + */ + Index: kernel/generic/include/synch/condvar.h =================================================================== --- kernel/generic/include/synch/condvar.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/synch/condvar.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -39,4 +39,5 @@ #include #include +#include #include @@ -55,4 +56,9 @@ extern int _condvar_wait_timeout(condvar_t *cv, mutex_t *mtx, uint32_t usec, int flags); +extern int _condvar_wait_timeout_spinlock(condvar_t *cv, spinlock_t *lock, + uint32_t usec, int flags); +extern int _condvar_wait_timeout_irq_spinlock(condvar_t *cv, + irq_spinlock_t *irq_lock, uint32_t usec, int flags); + #endif Index: kernel/generic/include/synch/rcu.h =================================================================== --- kernel/generic/include/synch/rcu.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/synch/rcu.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup sync + * @{ + */ +/** @file + */ + +#ifndef KERN_RCU_H_ +#define KERN_RCU_H_ + +#include +#include + + + + +/** Use to assign a pointer to newly initialized data to a rcu reader + * accessible pointer. + * + * Example: + * @code + * typedef struct exam { + * struct exam *next; + * int grade; + * } exam_t; + * + * exam_t *exam_list; + * // .. + * + * // Insert at the beginning of the list. + * exam_t *my_exam = malloc(sizeof(exam_t), 0); + * my_exam->grade = 5; + * my_exam->next = exam_list; + * rcu_assign(exam_list, my_exam); + * + * // Changes properly propagate. Every reader either sees + * // the old version of exam_list or the new version with + * // the fully initialized my_exam. + * rcu_synchronize(); + * // Now we can be sure every reader sees my_exam. + * + * @endcode + */ +#define rcu_assign(ptr, value) \ + do { \ + memory_barrier(); \ + (ptr) = (value); \ + } while (0) + +/** Use to access RCU protected data in a reader section. + * + * Example: + * @code + * exam_t *exam_list; + * // ... + * + * rcu_read_lock(); + * exam_t *first_exam = rcu_access(exam_list); + * // We can now safely use first_exam, it won't change + * // under us while we're using it. + * + * // .. + * rcu_read_unlock(); + * @endcode + */ +#define rcu_access(ptr) ACCESS_ONCE(ptr) + + + + +#include +#include +#include +#include + + +extern bool rcu_read_locked(void); +extern void rcu_synchronize(void); +extern void rcu_synchronize_expedite(void); +extern void rcu_call(rcu_item_t *rcu_item, rcu_func_t func); +extern void rcu_barrier(void); + +extern void rcu_print_stat(void); + +extern void rcu_init(void); +extern void rcu_stop(void); +extern void rcu_cpu_init(void); +extern void rcu_kinit_init(void); +extern void rcu_thread_init(struct thread*); +extern void rcu_thread_exiting(void); +extern void rcu_after_thread_ran(void); +extern void rcu_before_thread_runs(void); + +extern uint64_t rcu_completed_gps(void); +extern void _rcu_call(bool expedite, rcu_item_t *rcu_item, rcu_func_t func); +extern void _rcu_synchronize(bool expedite); + + +#ifdef RCU_PREEMPT_A + +#define RCU_CNT_INC (1 << 1) +#define RCU_WAS_PREEMPTED (1 << 0) + +/* Fwd. decl. because of inlining. */ +void _rcu_preempted_unlock(void); + +/** Delimits the start of an RCU reader critical section. + * + * Reader sections may be nested and are preemptable. You must not + * however block/sleep within reader sections. + */ +static inline void rcu_read_lock(void) +{ + THE->rcu_nesting += RCU_CNT_INC; +} + +/** Delimits the end of an RCU reader critical section. */ +static inline void rcu_read_unlock(void) +{ + THE->rcu_nesting -= RCU_CNT_INC; + + if (RCU_WAS_PREEMPTED == THE->rcu_nesting) { + _rcu_preempted_unlock(); + } +} + +#elif defined(RCU_PREEMPT_PODZIMEK) + +/* Fwd decl. required by the inlined implementation. Not part of public API. */ +extern rcu_gp_t _rcu_cur_gp; +extern void _rcu_signal_read_unlock(void); + + +/** Unconditionally records a quiescent state for the local cpu. */ +static inline void _rcu_record_qs(void) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + + /* + * A new GP was started since the last time we passed a QS. + * Notify the detector we have reached a new QS. + */ + if (CPU->rcu.last_seen_gp != _rcu_cur_gp) { + rcu_gp_t cur_gp = ACCESS_ONCE(_rcu_cur_gp); + /* + * Contain memory accesses within a reader critical section. + * If we are in rcu_lock() it also makes changes prior to the + * start of the GP visible in the reader section. + */ + memory_barrier(); + /* + * Acknowledge we passed a QS since the beginning of rcu.cur_gp. + * Cache coherency will lazily transport the value to the + * detector while it sleeps in gp_sleep(). + * + * Note that there is a theoretical possibility that we + * overwrite a more recent/greater last_seen_gp here with + * an older/smaller value. If this cpu is interrupted here + * while in rcu_lock() reader sections in the interrupt handler + * will update last_seen_gp to the same value as is currently + * in local cur_gp. However, if the cpu continues processing + * interrupts and the detector starts a new GP immediately, + * local interrupt handlers may update last_seen_gp again (ie + * properly ack the new GP) with a value greater than local cur_gp. + * Resetting last_seen_gp to a previous value here is however + * benign and we only have to remember that this reader may end up + * in cur_preempted even after the GP ends. That is why we + * append next_preempted to cur_preempted rather than overwriting + * it as if cur_preempted were empty. + */ + CPU->rcu.last_seen_gp = cur_gp; + } +} + +/** Delimits the start of an RCU reader critical section. + * + * Reader sections may be nested and are preemptable. You must not + * however block/sleep within reader sections. + */ +static inline void rcu_read_lock(void) +{ + ASSERT(CPU); + preemption_disable(); + + /* Record a QS if not in a reader critical section. */ + if (0 == CPU->rcu.nesting_cnt) + _rcu_record_qs(); + + ++CPU->rcu.nesting_cnt; + + preemption_enable(); +} + +/** Delimits the end of an RCU reader critical section. */ +static inline void rcu_read_unlock(void) +{ + ASSERT(CPU); + preemption_disable(); + + if (0 == --CPU->rcu.nesting_cnt) { + _rcu_record_qs(); + + /* + * The thread was preempted while in a critical section or + * the detector is eagerly waiting for this cpu's reader to finish. + */ + if (CPU->rcu.signal_unlock) { + /* Rechecks with disabled interrupts. */ + _rcu_signal_read_unlock(); + } + } + + preemption_enable(); +} +#endif + +#endif + +/** @} + */ Index: kernel/generic/include/synch/rcu_types.h =================================================================== --- kernel/generic/include/synch/rcu_types.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/synch/rcu_types.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup sync + * @{ + */ +/** @file + */ + +#ifndef KERN_RCU_TYPES_H_ +#define KERN_RCU_TYPES_H_ + +#include +#include + +#if !defined(RCU_PREEMPT_PODZIMEK) && !defined(RCU_PREEMPT_A) +#define RCU_PREEMPT_A +//#error You must select an RCU algorithm. +#endif + + +/* Fwd decl. */ +struct thread; +struct rcu_item; + +/** Grace period number typedef. */ +typedef uint64_t rcu_gp_t; + +/** RCU callback type. The passed rcu_item_t maybe freed. */ +typedef void (*rcu_func_t)(struct rcu_item *rcu_item); + +typedef struct rcu_item { + rcu_func_t func; + struct rcu_item *next; +} rcu_item_t; + + +/** RCU related per-cpu data. */ +typedef struct rcu_cpu_data { + /** The cpu recorded a quiescent state last time during this grace period.*/ + rcu_gp_t last_seen_gp; + +#ifdef RCU_PREEMPT_PODZIMEK + /** This cpu has not yet passed a quiescent state and it is delaying the + * detector. Once it reaches a QS it must sema_up(rcu.remaining_readers). + */ + bool is_delaying_gp; + + /** True if we should signal the detector that we exited a reader section. + * + * Equal to (THREAD->rcu.was_preempted || CPU->rcu.is_delaying_gp). + */ + bool signal_unlock; + + /** The number of times an RCU reader section is nested on this cpu. + * + * If positive, it is definitely executing reader code. If zero, + * the thread might already be executing reader code thanks to + * cpu instruction reordering. + */ + size_t nesting_cnt; +#endif + + /** Callbacks to invoke once the current grace period ends, ie cur_cbs_gp. + * Accessed by the local reclaimer only. + */ + rcu_item_t *cur_cbs; + /** Number of callbacks in cur_cbs. */ + size_t cur_cbs_cnt; + /** Callbacks to invoke once the next grace period ends, ie next_cbs_gp. + * Accessed by the local reclaimer only. + */ + rcu_item_t *next_cbs; + /** Number of callbacks in next_cbs. */ + size_t next_cbs_cnt; + /** New callbacks are place at the end of this list. */ + rcu_item_t *arriving_cbs; + /** Tail of arriving_cbs list. Disable interrupts to access. */ + rcu_item_t **parriving_cbs_tail; + /** Number of callbacks currently in arriving_cbs. + * Disable interrupts to access. + */ + size_t arriving_cbs_cnt; + + /** At the end of this grace period callbacks in cur_cbs will be invoked.*/ + rcu_gp_t cur_cbs_gp; + /** At the end of this grace period callbacks in next_cbs will be invoked. + * + * Should be the next grace period but it allows the reclaimer to + * notice if it missed a grace period end announcement. In that + * case it can execute next_cbs without waiting for another GP. + * + * Invariant: next_cbs_gp >= cur_cbs_gp + */ + rcu_gp_t next_cbs_gp; + + /** Positive if there are callbacks pending in arriving_cbs. */ + semaphore_t arrived_flag; + + /** The reclaimer should expedite GPs for cbs in arriving_cbs. */ + bool expedite_arriving; + + /** Protected by global rcu.barrier_mtx. */ + rcu_item_t barrier_item; + + /** Interruptable attached reclaimer thread. */ + struct thread *reclaimer_thr; + + /* Some statistics. */ + size_t stat_max_cbs; + size_t stat_avg_cbs; + size_t stat_missed_gps; + size_t stat_missed_gp_in_wait; + size_t stat_max_slice_cbs; + size_t last_arriving_cnt; +} rcu_cpu_data_t; + + +/** RCU related per-thread data. */ +typedef struct rcu_thread_data { + /** + * Nesting count of the thread's RCU read sections when the thread + * is not running. + */ + size_t nesting_cnt; + +#ifdef RCU_PREEMPT_PODZIMEK + + /** True if the thread was preempted in a reader section. + * + * The thread is place into rcu.cur_preempted or rcu.next_preempted + * and must remove itself in rcu_read_unlock(). + * + * Access with interrupts disabled. + */ + bool was_preempted; +#endif + + /** Preempted threads link. Access with rcu.prempt_lock.*/ + link_t preempt_link; +} rcu_thread_data_t; + + +#endif + +/** @} + */ Index: kernel/generic/include/synch/semaphore.h =================================================================== --- kernel/generic/include/synch/semaphore.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/synch/semaphore.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -53,4 +53,8 @@ _semaphore_down_timeout((s), (usec), SYNCH_FLAGS_NONE) +#define semaphore_down_interruptable(s) \ + (ESYNCH_INTERRUPTED != _semaphore_down_timeout((s), SYNCH_NO_TIMEOUT, \ + SYNCH_FLAGS_INTERRUPTIBLE)) + extern void semaphore_initialize(semaphore_t *, int); extern int _semaphore_down_timeout(semaphore_t *, uint32_t, unsigned int); Index: kernel/generic/include/synch/waitq.h =================================================================== --- kernel/generic/include/synch/waitq.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/include/synch/waitq.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -77,4 +77,5 @@ extern void waitq_sleep_finish(waitq_t *, int, ipl_t); extern void waitq_wakeup(waitq_t *, wakeup_mode_t); +extern void waitq_complete_wakeup(waitq_t *); extern void _waitq_wakeup_unsafe(waitq_t *, wakeup_mode_t); extern void waitq_interrupt_sleep(struct thread *); Index: kernel/generic/include/synch/workqueue.h =================================================================== --- kernel/generic/include/synch/workqueue.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/include/synch/workqueue.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ +/** @file + */ + +#ifndef KERN_WORKQUEUE_H_ +#define KERN_WORKQUEUE_H_ + +#include +#include + +/* Fwd decl. */ +struct work_item; +struct work_queue; +typedef struct work_queue work_queue_t; + +typedef void (*work_func_t)(struct work_item *); + +typedef struct work_item { + link_t queue_link; + work_func_t func; + +#ifdef CONFIG_DEBUG + /* Magic number for integrity checks. */ + uint32_t cookie; +#endif +} work_t; + + + +extern void workq_global_init(void); +extern void workq_global_worker_init(void); +extern void workq_global_stop(void); +extern int workq_global_enqueue_noblock(work_t *, work_func_t); +extern int workq_global_enqueue(work_t *, work_func_t); + +extern struct work_queue * workq_create(const char *); +extern void workq_destroy(struct work_queue *); +extern int workq_init(struct work_queue *, const char *); +extern void workq_stop(struct work_queue *); +extern int workq_enqueue_noblock(struct work_queue *, work_t *, work_func_t); +extern int workq_enqueue(struct work_queue *, work_t *, work_func_t); + +extern void workq_print_info(struct work_queue *); +extern void workq_global_print_info(void); + + +extern void workq_after_thread_ran(void); +extern void workq_before_thread_is_ready(thread_t *); + +#endif /* KERN_WORKQUEUE_H_ */ + +/** @} + */ Index: kernel/generic/src/adt/cht.c =================================================================== --- kernel/generic/src/adt/cht.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/src/adt/cht.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,1986 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/** @addtogroup genericadt + * @{ + */ + +/** + * @file + * @brief Scalable resizable concurrent lock-free hash table. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Logarithm of the min bucket count. Must be at least 3. 2^6 == 64 buckets. */ +#define CHT_MIN_ORDER 6 +/* Logarithm of the max bucket count. */ +#define CHT_MAX_ORDER (8 * sizeof(size_t)) +/* Minimum number of hash table buckets. */ +#define CHT_MIN_BUCKET_CNT (1 << CHT_MIN_ORDER) +/* Does not have to be a power of 2. */ +#define CHT_MAX_LOAD 2 + +typedef cht_ptr_t marked_ptr_t; +typedef bool (*equal_pred_t)(void *arg, const cht_link_t *item); + +typedef enum mark { + N_NORMAL = 0, + N_DELETED = 1, + N_CONST = 1, + N_INVALID = 3, + N_JOIN = 2, + N_JOIN_FOLLOWS = 2, + N_MARK_MASK = 3 +} mark_t; + +typedef enum walk_mode { + WM_NORMAL = 4, + WM_LEAVE_JOIN, + WM_MOVE_JOIN_FOLLOWS +} walk_mode_t; + +typedef struct wnd { + marked_ptr_t *ppred; + cht_link_t *cur; + cht_link_t *last; +} wnd_t; + + +/* Sentinel node used by all buckets. Stores the greatest possible hash value.*/ +static const cht_link_t sentinel = { + .link = 0, + .hash = -1 +}; + + +static size_t size_to_order(size_t bucket_cnt, size_t min_order); +static cht_buckets_t *alloc_buckets(size_t order, bool set_invalid); +static inline cht_link_t *find_lazy(cht_t *h, void *key); +static cht_link_t *search_bucket(cht_t *h, marked_ptr_t head, void *key, + size_t search_hash); +static cht_link_t *find_resizing(cht_t *h, void *key, size_t hash, + marked_ptr_t old_head, size_t old_idx); +static bool insert_impl(cht_t *h, cht_link_t *item, bool unique); +static bool insert_at(cht_link_t *item, const wnd_t *wnd, walk_mode_t walk_mode, + bool *resizing); +static bool has_duplicates(cht_t *h, const cht_link_t *item, size_t hash, + const wnd_t *cwnd); +static cht_link_t *find_duplicate(cht_t *h, const cht_link_t *item, size_t hash, + cht_link_t *start); +static bool remove_pred(cht_t *h, size_t hash, equal_pred_t pred, void *pred_arg); +static bool delete_at(cht_t *h, wnd_t *wnd, walk_mode_t walk_mode, + bool *deleted_but_gc, bool *resizing); +static bool mark_deleted(cht_link_t *cur, walk_mode_t walk_mode, bool *resizing); +static bool unlink_from_pred(wnd_t *wnd, walk_mode_t walk_mode, bool *resizing); +static bool find_wnd_and_gc_pred(cht_t *h, size_t hash, walk_mode_t walk_mode, + equal_pred_t pred, void *pred_arg, wnd_t *wnd, bool *resizing); +static bool find_wnd_and_gc(cht_t *h, size_t hash, walk_mode_t walk_mode, + wnd_t *wnd, bool *resizing); +static bool gc_deleted_node(cht_t *h, walk_mode_t walk_mode, wnd_t *wnd, + bool *resizing); +static bool join_completed(cht_t *h, const wnd_t *wnd); +static void upd_resizing_head(cht_t *h, size_t hash, marked_ptr_t **phead, + bool *join_finishing, walk_mode_t *walk_mode); +static void item_removed(cht_t *h); +static void item_inserted(cht_t *h); +static void free_later(cht_t *h, cht_link_t *item); +static void help_head_move(marked_ptr_t *psrc_head, marked_ptr_t *pdest_head); +static void start_head_move(marked_ptr_t *psrc_head); +static void mark_const(marked_ptr_t *psrc_head); +static void complete_head_move(marked_ptr_t *psrc_head, marked_ptr_t *pdest_head); +static void split_bucket(cht_t *h, marked_ptr_t *psrc_head, + marked_ptr_t *pdest_head, size_t split_hash); +static void mark_join_follows(cht_t *h, marked_ptr_t *psrc_head, + size_t split_hash, wnd_t *wnd); +static void mark_join_node(cht_link_t *join_node); +static void join_buckets(cht_t *h, marked_ptr_t *psrc_head, + marked_ptr_t *pdest_head, size_t split_hash); +static void link_to_join_node(cht_t *h, marked_ptr_t *pdest_head, + cht_link_t *join_node, size_t split_hash); +static void resize_table(work_t *arg); +static void grow_table(cht_t *h); +static void shrink_table(cht_t *h); +static void cleanup_join_node(cht_t *h, marked_ptr_t *new_head); +static void clear_join_and_gc(cht_t *h, cht_link_t *join_node, + marked_ptr_t *new_head); +static void cleanup_join_follows(cht_t *h, marked_ptr_t *new_head); +static marked_ptr_t make_link(const cht_link_t *next, mark_t mark); +static cht_link_t * get_next(marked_ptr_t link); +static mark_t get_mark(marked_ptr_t link); +static void next_wnd(wnd_t *wnd); +static bool same_node_pred(void *node, const cht_link_t *item2); +static size_t calc_key_hash(cht_t *h, void *key); +static size_t node_hash(cht_t *h, const cht_link_t *item); +static size_t calc_node_hash(cht_t *h, const cht_link_t *item); +static void memoize_node_hash(cht_t *h, cht_link_t *item); +static size_t calc_split_hash(size_t split_idx, size_t order); +static size_t calc_bucket_idx(size_t hash, size_t order); +static size_t grow_to_split_idx(size_t old_idx); +static size_t grow_idx(size_t idx); +static size_t shrink_idx(size_t idx); +static marked_ptr_t cas_link(marked_ptr_t *link, const cht_link_t *cur_next, + mark_t cur_mark, const cht_link_t *new_next, mark_t new_mark); +static marked_ptr_t _cas_link(marked_ptr_t *link, marked_ptr_t cur, + marked_ptr_t new); +static void cas_order_barrier(void); + + +bool cht_create(cht_t *h, size_t init_size, size_t min_size, size_t max_load, + cht_ops_t *op) +{ + ASSERT(h); + ASSERT(op && op->hash && op->key_hash && op->equal && op->key_equal); + /* Memoized hashes are stored in the rcu_link.func function pointer. */ + ASSERT(sizeof(size_t) == sizeof(rcu_func_t)); + ASSERT(sentinel.hash == (uintptr_t)sentinel.rcu_link.func); + + /* All operations are compulsory. */ + if (!op || !op->hash || !op->key_hash || !op->equal || !op->key_equal) + return false; + + size_t min_order = size_to_order(min_size, CHT_MIN_ORDER); + size_t order = size_to_order(init_size, min_order); + + h->b = alloc_buckets(order, false); + + if (!h->b) + return false; + + h->max_load = (max_load == 0) ? CHT_MAX_LOAD : max_load; + h->min_order = min_order; + h->new_b = 0; + h->op = op; + atomic_set(&h->item_cnt, 0); + atomic_set(&h->resize_reqs, 0); + /* + * Cached item hashes are stored in item->rcu_link.func. Once the item + * is deleted rcu_link.func will contain the value of invalid_hash. + */ + h->invalid_hash = (uintptr_t)h->op->remove_callback; + + /* Ensure the initialization takes place before we start using the table. */ + write_barrier(); + + return true; +} + +static cht_buckets_t *alloc_buckets(size_t order, bool set_invalid) +{ + size_t bucket_cnt = (1 << order); + size_t bytes = + sizeof(cht_buckets_t) + (bucket_cnt - 1) * sizeof(marked_ptr_t); + cht_buckets_t *b = malloc(bytes, FRAME_ATOMIC); + + if (!b) + return 0; + + b->order = order; + + marked_ptr_t head_link = set_invalid + ? make_link(&sentinel, N_INVALID) + : make_link(&sentinel, N_NORMAL); + + for (size_t i = 0; i < bucket_cnt; ++i) { + b->head[i] = head_link; + } + + return b; +} + +static size_t size_to_order(size_t bucket_cnt, size_t min_order) +{ + size_t order = min_order; + + /* Find a power of two such that bucket_cnt <= 2^order */ + do { + if (bucket_cnt <= ((size_t)1 << order)) + return order; + + ++order; + } while (order < CHT_MAX_ORDER); + + return order; +} + + +void cht_destroy(cht_t *h) +{ + /* Wait for resize to complete. */ + while (0 < atomic_get(&h->resize_reqs)) { + rcu_barrier(); + } + + /* Wait for all remove_callback()s to complete. */ + rcu_barrier(); + + free(h->b); + h->b = 0; +} + +cht_link_t *cht_find(cht_t *h, void *key) +{ + /* Make the most recent changes to the table visible. */ + read_barrier(); + return cht_find_lazy(h, key); +} + +cht_link_t *cht_find_lazy(cht_t *h, void *key) +{ + return find_lazy(h, key); +} + +static inline cht_link_t *find_lazy(cht_t *h, void *key) +{ + ASSERT(h); + ASSERT(rcu_read_locked()); + + size_t hash = calc_key_hash(h, key); + + cht_buckets_t *b = rcu_access(h->b); + size_t idx = calc_bucket_idx(hash, b->order); + /* + * No need for access_once. b->head[idx] will point to an allocated node + * even if marked invalid until we exit rcu read section. + */ + marked_ptr_t head = b->head[idx]; + + if (N_INVALID == get_mark(head)) + return find_resizing(h, key, hash, head, idx); + + return search_bucket(h, head, key, hash); +} + +cht_link_t *cht_find_next(cht_t *h, const cht_link_t *item) +{ + /* Make the most recent changes to the table visible. */ + read_barrier(); + return cht_find_next_lazy(h, item); +} + +cht_link_t *cht_find_next_lazy(cht_t *h, const cht_link_t *item) +{ + ASSERT(h); + ASSERT(rcu_read_locked()); + ASSERT(item); + + return find_duplicate(h, item, calc_node_hash(h, item), get_next(item->link)); +} + +static inline cht_link_t *search_bucket(cht_t *h, marked_ptr_t head, void *key, + size_t search_hash) +{ + /* + * It is safe to access nodes even outside of this bucket (eg when + * splitting the bucket). The resizer makes sure that any node we + * may find by following the next pointers is allocated. + */ + + cht_link_t *cur = 0; + marked_ptr_t prev = head; + +try_again: + /* Filter out items with different hashes. */ + do { + cur = get_next(prev); + ASSERT(cur); + prev = cur->link; + } while (node_hash(h, cur) < search_hash); + + /* + * Only search for an item with an equal key if cur is not the sentinel + * node or a node with a different hash. + */ + while (node_hash(h, cur) == search_hash) { + if (h->op->key_equal(key, cur)) { + if (!(N_DELETED & get_mark(cur->link))) + return cur; + } + + cur = get_next(cur->link); + ASSERT(cur); + } + + /* + * In the unlikely case that we have encountered a node whose cached + * hash has been overwritten due to a pending rcu_call for it, skip + * the node and try again. + */ + if (node_hash(h, cur) == h->invalid_hash) { + prev = cur->link; + goto try_again; + } + + return 0; +} + +static cht_link_t *find_resizing(cht_t *h, void *key, size_t hash, + marked_ptr_t old_head, size_t old_idx) +{ + ASSERT(N_INVALID == get_mark(old_head)); + ASSERT(h->new_b); + + size_t new_idx = calc_bucket_idx(hash, h->new_b->order); + marked_ptr_t new_head = h->new_b->head[new_idx]; + marked_ptr_t search_head = new_head; + + /* Growing. */ + if (h->b->order < h->new_b->order) { + /* + * Old bucket head is invalid, so it must have been already + * moved. Make the new head visible if still not visible, ie + * invalid. + */ + if (N_INVALID == get_mark(new_head)) { + /* + * We should be searching a newly added bucket but the old + * moved bucket has not yet been split (its marked invalid) + * or we have not yet seen the split. + */ + if (grow_idx(old_idx) != new_idx) { + /* + * Search the moved bucket. It is guaranteed to contain + * items of the newly added bucket that were present + * before the moved bucket was split. + */ + new_head = h->new_b->head[grow_idx(old_idx)]; + } + + /* new_head is now the moved bucket, either valid or invalid. */ + + /* + * The old bucket was definitely moved to new_head but the + * change of new_head had not yet propagated to this cpu. + */ + if (N_INVALID == get_mark(new_head)) { + /* + * We could issue a read_barrier() and make the now valid + * moved bucket head new_head visible, but instead fall back + * on using the old bucket. Although the old bucket head is + * invalid, it points to a node that is allocated and in the + * right bucket. Before the node can be freed, it must be + * unlinked from the head (or another item after that item + * modified the new_head) and a grace period must elapse. + * As a result had the node been already freed the grace + * period preceeding the free() would make the unlink and + * any changes to new_head visible. Therefore, it is safe + * to use the node pointed to from the old bucket head. + */ + + search_head = old_head; + } else { + search_head = new_head; + } + } + + return search_bucket(h, search_head, key, hash); + } else if (h->b->order > h->new_b->order) { + /* Shrinking. */ + + /* Index of the bucket in the old table that was moved. */ + size_t move_src_idx = grow_idx(new_idx); + marked_ptr_t moved_old_head = h->b->head[move_src_idx]; + + /* + * h->b->head[move_src_idx] had already been moved to new_head + * but the change to new_head had not yet propagated to us. + */ + if (N_INVALID == get_mark(new_head)) { + /* + * new_head is definitely valid and we could make it visible + * to this cpu with a read_barrier(). Instead, use the bucket + * in the old table that was moved even though it is now marked + * as invalid. The node it points to must be allocated because + * a grace period would have to elapse before it could be freed; + * and the grace period would make the now valid new_head + * visible to all cpus. + * + * Note that move_src_idx may not be the same as old_idx. + * If move_src_idx != old_idx then old_idx is the bucket + * in the old table that is not moved but instead it is + * appended to the moved bucket, ie it is added at the tail + * of new_head. In that case an invalid old_head notes that + * it had already been merged into (the moved) new_head. + * We will try to search that bucket first because it + * may contain some newly added nodes after the bucket + * join. Moreover, the bucket joining link may already be + * visible even if new_head is not. Therefore, if we're + * lucky we'll find the item via moved_old_head. In any + * case, we'll retry in proper old_head if not found. + */ + search_head = moved_old_head; + } + + cht_link_t *ret = search_bucket(h, search_head, key, hash); + + if (ret) + return ret; + /* + * Bucket old_head was already joined with moved_old_head + * in the new table but we have not yet seen change of the + * joining link (or the item is not in the table). + */ + if (move_src_idx != old_idx && get_next(old_head) != &sentinel) { + /* + * Note that old_head (the bucket to be merged into new_head) + * points to an allocated join node (if non-null) even if marked + * invalid. Before the resizer lets join nodes to be unlinked + * (and freed) it sets old_head to 0 and waits for a grace period. + * So either the invalid old_head points to join node; or old_head + * is null and we would have seen a completed bucket join while + * traversing search_head. + */ + ASSERT(N_JOIN & get_mark(get_next(old_head)->link)); + return search_bucket(h, old_head, key, hash); + } + + return 0; + } else { + /* + * Resize is almost done. The resizer is waiting to make + * sure all cpus see that the new table replaced the old one. + */ + ASSERT(h->b->order == h->new_b->order); + /* + * The resizer must ensure all new bucket heads are visible before + * replacing the old table. + */ + ASSERT(N_NORMAL == get_mark(new_head)); + return search_bucket(h, new_head, key, hash); + } +} + + +void cht_insert(cht_t *h, cht_link_t *item) +{ + insert_impl(h, item, false); +} + +bool cht_insert_unique(cht_t *h, cht_link_t *item) +{ + return insert_impl(h, item, true); +} + +static bool insert_impl(cht_t *h, cht_link_t *item, bool unique) +{ + rcu_read_lock(); + + cht_buckets_t *b = rcu_access(h->b); + memoize_node_hash(h, item); + size_t hash = node_hash(h, item); + size_t idx = calc_bucket_idx(hash, b->order); + marked_ptr_t *phead = &b->head[idx]; + + bool resizing = false; + bool inserted = false; + + do { + walk_mode_t walk_mode = WM_NORMAL; + bool join_finishing; + + resizing = resizing || (N_NORMAL != get_mark(*phead)); + + /* The table is resizing. Get the correct bucket head. */ + if (resizing) { + upd_resizing_head(h, hash, &phead, &join_finishing, &walk_mode); + } + + wnd_t wnd = { + .ppred = phead, + .cur = get_next(*phead), + .last = 0 + }; + + if (!find_wnd_and_gc(h, hash, walk_mode, &wnd, &resizing)) { + /* Could not GC a node; or detected an unexpected resize. */ + continue; + } + + if (unique && has_duplicates(h, item, hash, &wnd)) { + rcu_read_unlock(); + return false; + } + + inserted = insert_at(item, &wnd, walk_mode, &resizing); + } while (!inserted); + + rcu_read_unlock(); + + item_inserted(h); + return true; +} + +inline static bool insert_at(cht_link_t *item, const wnd_t *wnd, + walk_mode_t walk_mode, bool *resizing) +{ + marked_ptr_t ret; + + if (walk_mode == WM_NORMAL) { + item->link = make_link(wnd->cur, N_NORMAL); + /* Initialize the item before adding it to a bucket. */ + memory_barrier(); + + /* Link a clean/normal predecessor to the item. */ + ret = cas_link(wnd->ppred, wnd->cur, N_NORMAL, item, N_NORMAL); + + if (ret == make_link(wnd->cur, N_NORMAL)) { + return true; + } else { + /* This includes an invalid head but not a const head. */ + *resizing = ((N_JOIN_FOLLOWS | N_JOIN) & get_mark(ret)); + return false; + } + } else if (walk_mode == WM_MOVE_JOIN_FOLLOWS) { + /* Move JOIN_FOLLOWS mark but filter out the DELETED mark. */ + mark_t jf_mark = get_mark(*wnd->ppred) & N_JOIN_FOLLOWS; + item->link = make_link(wnd->cur, jf_mark); + /* Initialize the item before adding it to a bucket. */ + memory_barrier(); + + /* Link the not-deleted predecessor to the item. Move its JF mark. */ + ret = cas_link(wnd->ppred, wnd->cur, jf_mark, item, N_NORMAL); + + return ret == make_link(wnd->cur, jf_mark); + } else { + ASSERT(walk_mode == WM_LEAVE_JOIN); + + item->link = make_link(wnd->cur, N_NORMAL); + /* Initialize the item before adding it to a bucket. */ + memory_barrier(); + + mark_t pred_mark = get_mark(*wnd->ppred); + /* If the predecessor is a join node it may be marked deleted.*/ + mark_t exp_pred_mark = (N_JOIN & pred_mark) ? pred_mark : N_NORMAL; + + ret = cas_link(wnd->ppred, wnd->cur, exp_pred_mark, item, exp_pred_mark); + return ret == make_link(wnd->cur, exp_pred_mark); + } +} + +static inline bool has_duplicates(cht_t *h, const cht_link_t *item, size_t hash, + const wnd_t *wnd) +{ + ASSERT(wnd->cur); + ASSERT(wnd->cur == &sentinel || hash <= node_hash(h, wnd->cur) + || node_hash(h, wnd->cur) == h->invalid_hash); + + /* hash < node_hash(h, wnd->cur) */ + if (hash != node_hash(h, wnd->cur) && h->invalid_hash != node_hash(h, wnd->cur)) + return false; + + /* + * Load the most recent node marks. Otherwise we might pronounce a + * logically deleted node for a duplicate of the item just because + * the deleted node's DEL mark had not yet propagated to this cpu. + */ + read_barrier(); + return 0 != find_duplicate(h, item, hash, wnd->cur); +} + +static cht_link_t *find_duplicate(cht_t *h, const cht_link_t *item, size_t hash, + cht_link_t *start) +{ + ASSERT(hash <= node_hash(h, start) || h->invalid_hash == node_hash(h, start)); + + cht_link_t *cur = start; + +try_again: + ASSERT(cur); + + while (node_hash(h, cur) == hash) { + ASSERT(cur != &sentinel); + + bool deleted = (N_DELETED & get_mark(cur->link)); + + /* Skip logically deleted nodes. */ + if (!deleted && h->op->equal(item, cur)) + return cur; + + cur = get_next(cur->link); + ASSERT(cur); + } + + if (h->invalid_hash == node_hash(h, cur)) { + cur = get_next(cur->link); + goto try_again; + } + + return 0; +} + +size_t cht_remove_key(cht_t *h, void *key) +{ + ASSERT(h); + + size_t hash = calc_key_hash(h, key); + size_t removed = 0; + + while (remove_pred(h, hash, h->op->key_equal, key)) + ++removed; + + return removed; +} + +bool cht_remove_item(cht_t *h, cht_link_t *item) +{ + ASSERT(h); + ASSERT(item); + + /* + * Even though we know the node we want to delete we must unlink it + * from the correct bucket and from a clean/normal predecessor. Therefore, + * we search for it again from the beginning of the correct bucket. + */ + size_t hash = calc_node_hash(h, item); + return remove_pred(h, hash, same_node_pred, item); +} + + +static bool remove_pred(cht_t *h, size_t hash, equal_pred_t pred, void *pred_arg) +{ + rcu_read_lock(); + + bool resizing = false; + bool deleted = false; + bool deleted_but_gc = false; + + cht_buckets_t *b = rcu_access(h->b); + size_t idx = calc_bucket_idx(hash, b->order); + marked_ptr_t *phead = &b->head[idx]; + + do { + walk_mode_t walk_mode = WM_NORMAL; + bool join_finishing = false; + + resizing = resizing || (N_NORMAL != get_mark(*phead)); + + /* The table is resizing. Get the correct bucket head. */ + if (resizing) { + upd_resizing_head(h, hash, &phead, &join_finishing, &walk_mode); + } + + wnd_t wnd = { + .ppred = phead, + .cur = get_next(*phead), + .last = 0 + }; + + if (!find_wnd_and_gc_pred( + h, hash, walk_mode, pred, pred_arg, &wnd, &resizing)) { + /* Could not GC a node; or detected an unexpected resize. */ + continue; + } + + /* + * The item lookup is affected by a bucket join but effects of + * the bucket join have not been seen while searching for the item. + */ + if (join_finishing && !join_completed(h, &wnd)) { + /* + * Bucket was appended at the end of another but the next + * ptr linking them together was not visible on this cpu. + * join_completed() makes this appended bucket visible. + */ + continue; + } + + /* Already deleted, but delete_at() requested one GC pass. */ + if (deleted_but_gc) + break; + + bool found = (wnd.cur != &sentinel && pred(pred_arg, wnd.cur)); + + if (!found) { + rcu_read_unlock(); + return false; + } + + deleted = delete_at(h, &wnd, walk_mode, &deleted_but_gc, &resizing); + } while (!deleted || deleted_but_gc); + + rcu_read_unlock(); + return true; +} + + +static inline bool delete_at(cht_t *h, wnd_t *wnd, walk_mode_t walk_mode, + bool *deleted_but_gc, bool *resizing) +{ + ASSERT(wnd->cur && wnd->cur != &sentinel); + + *deleted_but_gc = false; + + if (!mark_deleted(wnd->cur, walk_mode, resizing)) { + /* Already deleted, or unexpectedly marked as JOIN/JOIN_FOLLOWS. */ + return false; + } + + /* Marked deleted. Unlink from the bucket. */ + + /* Never unlink join nodes. */ + if (walk_mode == WM_LEAVE_JOIN && (N_JOIN & get_mark(wnd->cur->link))) + return true; + + cas_order_barrier(); + + if (unlink_from_pred(wnd, walk_mode, resizing)) { + free_later(h, wnd->cur); + } else { + *deleted_but_gc = true; + } + + return true; +} + +static inline bool mark_deleted(cht_link_t *cur, walk_mode_t walk_mode, + bool *resizing) +{ + ASSERT(cur && cur != &sentinel); + + /* + * Btw, we could loop here if the cas fails but let's not complicate + * things and let's retry from the head of the bucket. + */ + + cht_link_t *next = get_next(cur->link); + + if (walk_mode == WM_NORMAL) { + /* Only mark clean/normal nodes - JF/JN is used only during resize. */ + marked_ptr_t ret = cas_link(&cur->link, next, N_NORMAL, next, N_DELETED); + + if (ret != make_link(next, N_NORMAL)) { + *resizing = (N_JOIN | N_JOIN_FOLLOWS) & get_mark(ret); + return false; + } + } else { + ASSERT(N_JOIN == N_JOIN_FOLLOWS); + + /* Keep the N_JOIN/N_JOIN_FOLLOWS mark but strip N_DELETED. */ + mark_t cur_mark = get_mark(cur->link) & N_JOIN_FOLLOWS; + + marked_ptr_t ret = + cas_link(&cur->link, next, cur_mark, next, cur_mark | N_DELETED); + + if (ret != make_link(next, cur_mark)) + return false; + } + + return true; +} + +static inline bool unlink_from_pred(wnd_t *wnd, walk_mode_t walk_mode, + bool *resizing) +{ + ASSERT(wnd->cur != &sentinel); + ASSERT(wnd->cur && (N_DELETED & get_mark(wnd->cur->link))); + + cht_link_t *next = get_next(wnd->cur->link); + + if (walk_mode == WM_LEAVE_JOIN) { + /* Never try to unlink join nodes. */ + ASSERT(!(N_JOIN & get_mark(wnd->cur->link))); + + mark_t pred_mark = get_mark(*wnd->ppred); + /* Succeed only if the predecessor is clean/normal or a join node. */ + mark_t exp_pred_mark = (N_JOIN & pred_mark) ? pred_mark : N_NORMAL; + + marked_ptr_t pred_link = make_link(wnd->cur, exp_pred_mark); + marked_ptr_t next_link = make_link(next, exp_pred_mark); + + if (pred_link != _cas_link(wnd->ppred, pred_link, next_link)) + return false; + } else { + ASSERT(walk_mode == WM_MOVE_JOIN_FOLLOWS || walk_mode == WM_NORMAL); + /* Move the JF mark if set. Clear DEL mark. */ + mark_t cur_mark = N_JOIN_FOLLOWS & get_mark(wnd->cur->link); + + /* The predecessor must be clean/normal. */ + marked_ptr_t pred_link = make_link(wnd->cur, N_NORMAL); + /* Link to cur's successor keeping/copying cur's JF mark. */ + marked_ptr_t next_link = make_link(next, cur_mark); + + marked_ptr_t ret = _cas_link(wnd->ppred, pred_link, next_link); + + if (pred_link != ret) { + /* If we're not resizing the table there are no JF/JN nodes. */ + *resizing = (walk_mode == WM_NORMAL) + && (N_JOIN_FOLLOWS & get_mark(ret)); + return false; + } + } + + return true; +} + + +static bool find_wnd_and_gc_pred(cht_t *h, size_t hash, walk_mode_t walk_mode, + equal_pred_t pred, void *pred_arg, wnd_t *wnd, bool *resizing) +{ + ASSERT(wnd->cur); + + if (wnd->cur == &sentinel) + return true; + + /* + * A read barrier is not needed here to bring up the most recent + * node marks (esp the N_DELETED). At worst we'll try to delete + * an already deleted node; fail in delete_at(); and retry. + */ + + size_t cur_hash; + +try_again: + cur_hash = node_hash(h, wnd->cur); + + while (cur_hash <= hash) { + ASSERT(wnd->cur && wnd->cur != &sentinel); + + /* GC any deleted nodes on the way. */ + if (N_DELETED & get_mark(wnd->cur->link)) { + if (!gc_deleted_node(h, walk_mode, wnd, resizing)) { + /* Retry from the head of a bucket. */ + return false; + } + } else { + /* Is this the node we were looking for? */ + if (cur_hash == hash && pred(pred_arg, wnd->cur)) + return true; + + next_wnd(wnd); + } + + cur_hash = node_hash(h, wnd->cur); + } + + if (cur_hash == h->invalid_hash) { + next_wnd(wnd); + ASSERT(wnd->cur); + goto try_again; + } + + /* The searched for node is not in the current bucket. */ + return true; +} + +/* todo: comment different semantics (eg deleted JN first w/ specific hash) */ +static bool find_wnd_and_gc(cht_t *h, size_t hash, walk_mode_t walk_mode, + wnd_t *wnd, bool *resizing) +{ +try_again: + ASSERT(wnd->cur); + + while (node_hash(h, wnd->cur) < hash) { + /* GC any deleted nodes along the way to our desired node. */ + if (N_DELETED & get_mark(wnd->cur->link)) { + if (!gc_deleted_node(h, walk_mode, wnd, resizing)) { + /* Failed to remove the garbage node. Retry. */ + return false; + } + } else { + next_wnd(wnd); + } + + ASSERT(wnd->cur); + } + + if (node_hash(h, wnd->cur) == h->invalid_hash) { + next_wnd(wnd); + goto try_again; + } + + /* wnd->cur may be 0 or even marked N_DELETED. */ + return true; +} + +static bool gc_deleted_node(cht_t *h, walk_mode_t walk_mode, wnd_t *wnd, + bool *resizing) +{ + ASSERT(N_DELETED & get_mark(wnd->cur->link)); + + /* Skip deleted JOIN nodes. */ + if (walk_mode == WM_LEAVE_JOIN && (N_JOIN & get_mark(wnd->cur->link))) { + next_wnd(wnd); + } else { + /* Ordinary deleted node or a deleted JOIN_FOLLOWS. */ + ASSERT(walk_mode != WM_LEAVE_JOIN + || !((N_JOIN | N_JOIN_FOLLOWS) & get_mark(wnd->cur->link))); + + /* Unlink an ordinary deleted node, move JOIN_FOLLOWS mark. */ + if (!unlink_from_pred(wnd, walk_mode, resizing)) { + /* Retry. The predecessor was deleted, invalid, const, join_follows. */ + return false; + } + + free_later(h, wnd->cur); + + /* Leave ppred as is. */ + wnd->last = wnd->cur; + wnd->cur = get_next(wnd->cur->link); + } + + return true; +} + +static bool join_completed(cht_t *h, const wnd_t *wnd) +{ + /* + * The table is shrinking and the searched for item is in a bucket + * appended to another. Check that the link joining these two buckets + * is visible and if not, make it visible to this cpu. + */ + + /* + * Resizer ensures h->b->order stays the same for the duration of this + * func. We got here because there was an alternative head to search. + * The resizer waits for all preexisting readers to finish after + * it + */ + ASSERT(h->b->order > h->new_b->order); + ASSERT(wnd->cur); + + /* Either we did not need the joining link or we have already followed it.*/ + if (wnd->cur != &sentinel) + return true; + + /* We have reached the end of a bucket. */ + + if (wnd->last != &sentinel) { + size_t last_seen_hash = node_hash(h, wnd->last); + + if (last_seen_hash == h->invalid_hash) { + last_seen_hash = calc_node_hash(h, wnd->last); + } + + size_t last_old_idx = calc_bucket_idx(last_seen_hash, h->b->order); + size_t move_src_idx = grow_idx(shrink_idx(last_old_idx)); + + /* + * Last node seen was in the joining bucket - if the searched + * for node is there we will find it. + */ + if (move_src_idx != last_old_idx) + return true; + } + + /* + * Reached the end of the bucket but no nodes from the joining bucket + * were seen. There should have at least been a JOIN node so we have + * definitely not seen (and followed) the joining link. Make the link + * visible and retry. + */ + read_barrier(); + return false; +} + +static void upd_resizing_head(cht_t *h, size_t hash, marked_ptr_t **phead, + bool *join_finishing, walk_mode_t *walk_mode) +{ + cht_buckets_t *b = rcu_access(h->b); + size_t old_idx = calc_bucket_idx(hash, b->order); + size_t new_idx = calc_bucket_idx(hash, h->new_b->order); + + marked_ptr_t *pold_head = &b->head[old_idx]; + marked_ptr_t *pnew_head = &h->new_b->head[new_idx]; + + /* In any case, use the bucket in the new table. */ + *phead = pnew_head; + + /* Growing the table. */ + if (b->order < h->new_b->order) { + size_t move_dest_idx = grow_idx(old_idx); + marked_ptr_t *pmoved_head = &h->new_b->head[move_dest_idx]; + + /* Complete moving the bucket from the old to the new table. */ + help_head_move(pold_head, pmoved_head); + + /* The hash belongs to the moved bucket. */ + if (move_dest_idx == new_idx) { + ASSERT(pmoved_head == pnew_head); + /* + * move_head() makes the new head of the moved bucket visible. + * The new head may be marked with a JOIN_FOLLOWS + */ + ASSERT(!(N_CONST & get_mark(*pmoved_head))); + *walk_mode = WM_MOVE_JOIN_FOLLOWS; + } else { + ASSERT(pmoved_head != pnew_head); + /* + * The hash belongs to the bucket that is the result of splitting + * the old/moved bucket, ie the bucket that contains the second + * half of the split/old/moved bucket. + */ + + /* The moved bucket has not yet been split. */ + if (N_NORMAL != get_mark(*pnew_head)) { + size_t split_hash = calc_split_hash(new_idx, h->new_b->order); + split_bucket(h, pmoved_head, pnew_head, split_hash); + /* + * split_bucket() makes the new head visible. No + * JOIN_FOLLOWS in this part of split bucket. + */ + ASSERT(N_NORMAL == get_mark(*pnew_head)); + } + + *walk_mode = WM_LEAVE_JOIN; + } + } else if (h->new_b->order < b->order ) { + /* Shrinking the table. */ + + size_t move_src_idx = grow_idx(new_idx); + + /* + * Complete moving the bucket from the old to the new table. + * Makes a valid pnew_head visible if already moved. + */ + help_head_move(&b->head[move_src_idx], pnew_head); + + /* Hash belongs to the bucket to be joined with the moved bucket. */ + if (move_src_idx != old_idx) { + /* Bucket join not yet completed. */ + if (N_INVALID != get_mark(*pold_head)) { + size_t split_hash = calc_split_hash(old_idx, b->order); + join_buckets(h, pold_head, pnew_head, split_hash); + } + + /* + * The resizer sets pold_head to &sentinel when all cpus are + * guaranteed to see the bucket join. + */ + *join_finishing = (&sentinel != get_next(*pold_head)); + } + + /* move_head() or join_buckets() makes it so or makes the mark visible.*/ + ASSERT(N_INVALID == get_mark(*pold_head)); + /* move_head() makes it visible. No JOIN_FOLLOWS used when shrinking. */ + ASSERT(N_NORMAL == get_mark(*pnew_head)); + + *walk_mode = WM_LEAVE_JOIN; + } else { + /* + * Final stage of resize. The resizer is waiting for all + * readers to notice that the old table had been replaced. + */ + ASSERT(b == h->new_b); + *walk_mode = WM_NORMAL; + } +} + + +#if 0 +static void move_head(marked_ptr_t *psrc_head, marked_ptr_t *pdest_head) +{ + start_head_move(psrc_head); + cas_order_barrier(); + complete_head_move(psrc_head, pdest_head); +} +#endif + +static inline void help_head_move(marked_ptr_t *psrc_head, + marked_ptr_t *pdest_head) +{ + /* Head move has to in progress already when calling this func. */ + ASSERT(N_CONST & get_mark(*psrc_head)); + + /* Head already moved. */ + if (N_INVALID == get_mark(*psrc_head)) { + /* Effects of the head move have not yet propagated to this cpu. */ + if (N_INVALID == get_mark(*pdest_head)) { + /* Make the move visible on this cpu. */ + read_barrier(); + } + } else { + complete_head_move(psrc_head, pdest_head); + } + + ASSERT(!(N_CONST & get_mark(*pdest_head))); +} + +static void start_head_move(marked_ptr_t *psrc_head) +{ + /* Mark src head immutable. */ + mark_const(psrc_head); +} + +static void mark_const(marked_ptr_t *psrc_head) +{ + marked_ptr_t ret, src_link; + + /* Mark src head immutable. */ + do { + cht_link_t *next = get_next(*psrc_head); + src_link = make_link(next, N_NORMAL); + + /* Mark the normal/clean src link immutable/const. */ + ret = cas_link(psrc_head, next, N_NORMAL, next, N_CONST); + } while(ret != src_link && !(N_CONST & get_mark(ret))); +} + +static void complete_head_move(marked_ptr_t *psrc_head, marked_ptr_t *pdest_head) +{ + ASSERT(N_JOIN_FOLLOWS != get_mark(*psrc_head)); + ASSERT(N_CONST & get_mark(*psrc_head)); + + cht_link_t *next = get_next(*psrc_head); + marked_ptr_t ret; + + ret = cas_link(pdest_head, &sentinel, N_INVALID, next, N_NORMAL); + ASSERT(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret))); + cas_order_barrier(); + + ret = cas_link(psrc_head, next, N_CONST, next, N_INVALID); + ASSERT(ret == make_link(next, N_CONST) || (N_INVALID == get_mark(ret))); + cas_order_barrier(); +} + +static void split_bucket(cht_t *h, marked_ptr_t *psrc_head, + marked_ptr_t *pdest_head, size_t split_hash) +{ + /* Already split. */ + if (N_NORMAL == get_mark(*pdest_head)) + return; + + /* + * L == Last node of the first part of the split bucket. That part + * remains in the original/src bucket. + * F == First node of the second part of the split bucket. That part + * will be referenced from the dest bucket head. + * + * We want to first mark a clean L as JF so that updaters unaware of + * the split (or table resize): + * - do not insert a new node between L and F + * - do not unlink L (that is why it has to be clean/normal) + * - do not unlink F + * + * Then we can safely mark F as JN even if it has been marked deleted. + * Once F is marked as JN updaters aware of table resize will not + * attempt to unlink it (JN will have two predecessors - we cannot + * safely unlink from both at the same time). Updaters unaware of + * ongoing resize can reach F only via L and that node is already + * marked JF, so they won't unlink F. + * + * Last, link the new/dest head to F. + * + * + * 0) ,-- split_hash, first hash of the dest bucket + * v + * [src_head | N] -> .. -> [L] -> [F] + * [dest_head | Inv] + * + * 1) ,-- split_hash + * v + * [src_head | N] -> .. -> [JF] -> [F] + * [dest_head | Inv] + * + * 2) ,-- split_hash + * v + * [src_head | N] -> .. -> [JF] -> [JN] + * [dest_head | Inv] + * + * 2) ,-- split_hash + * v + * [src_head | N] -> .. -> [JF] -> [JN] + * ^ + * [dest_head | N] -----------------' + */ + wnd_t wnd; + + rcu_read_lock(); + + /* Mark the last node of the first part of the split bucket as JF. */ + mark_join_follows(h, psrc_head, split_hash, &wnd); + cas_order_barrier(); + + /* There are nodes in the dest bucket, ie the second part of the split. */ + if (wnd.cur != &sentinel) { + /* + * Mark the first node of the dest bucket as a join node so + * updaters do not attempt to unlink it if it is deleted. + */ + mark_join_node(wnd.cur); + cas_order_barrier(); + } else { + /* + * Second part of the split bucket is empty. There are no nodes + * to mark as JOIN nodes and there never will be. + */ + } + + /* Link the dest head to the second part of the split. */ + marked_ptr_t ret = + cas_link(pdest_head, &sentinel, N_INVALID, wnd.cur, N_NORMAL); + ASSERT(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret))); + cas_order_barrier(); + + rcu_read_unlock(); +} + +static void mark_join_follows(cht_t *h, marked_ptr_t *psrc_head, + size_t split_hash, wnd_t *wnd) +{ + /* See comment in split_bucket(). */ + + bool done; + do { + bool resizing = false; + wnd->ppred = psrc_head; + wnd->cur = get_next(*psrc_head); + + /* + * Find the split window, ie the last node of the first part of + * the split bucket and the its successor - the first node of + * the second part of the split bucket. Retry if GC failed. + */ + if (!find_wnd_and_gc(h, split_hash, WM_MOVE_JOIN_FOLLOWS, wnd, &resizing)) + continue; + + /* Must not report that the table is resizing if WM_MOVE_JOIN_FOLLOWS.*/ + ASSERT(!resizing); + /* + * Mark the last node of the first half of the split bucket + * that a join node follows. It must be clean/normal. + */ + marked_ptr_t ret + = cas_link(wnd->ppred, wnd->cur, N_NORMAL, wnd->cur, N_JOIN_FOLLOWS); + + /* + * Successfully marked as a JF node or already marked that way (even + * if also marked deleted - unlinking the node will move the JF mark). + */ + done = (ret == make_link(wnd->cur, N_NORMAL)) + || (N_JOIN_FOLLOWS & get_mark(ret)); + } while (!done); +} + +static void mark_join_node(cht_link_t *join_node) +{ + /* See comment in split_bucket(). */ + + bool done; + do { + cht_link_t *next = get_next(join_node->link); + mark_t mark = get_mark(join_node->link); + + /* + * May already be marked as deleted, but it won't be unlinked + * because its predecessor is marked with JOIN_FOLLOWS or CONST. + */ + marked_ptr_t ret + = cas_link(&join_node->link, next, mark, next, mark | N_JOIN); + + /* Successfully marked or already marked as a join node. */ + done = (ret == make_link(next, mark)) + || (N_JOIN & get_mark(ret)); + } while(!done); +} + + +static void join_buckets(cht_t *h, marked_ptr_t *psrc_head, + marked_ptr_t *pdest_head, size_t split_hash) +{ + /* Buckets already joined. */ + if (N_INVALID == get_mark(*psrc_head)) + return; + /* + * F == First node of psrc_head, ie the bucket we want to append + * to (ie join with) the bucket starting at pdest_head. + * L == Last node of pdest_head, ie the bucket that psrc_head will + * be appended to. + * + * (1) We first mark psrc_head immutable to signal that a join is + * in progress and so that updaters unaware of the join (or table + * resize): + * - do not insert new nodes between the head psrc_head and F + * - do not unlink F (it may already be marked deleted) + * + * (2) Next, F is marked as a join node. Updaters aware of table resize + * will not attempt to unlink it. We cannot safely/atomically unlink + * the join node because it will be pointed to from two different + * buckets. Updaters unaware of resize will fail to unlink the join + * node due to the head being marked immutable. + * + * (3) Then the tail of the bucket at pdest_head is linked to the join + * node. From now on, nodes in both buckets can be found via pdest_head. + * + * (4) Last, mark immutable psrc_head as invalid. It signals updaters + * that the join is complete and they can insert new nodes (originally + * destined for psrc_head) into pdest_head. + * + * Note that pdest_head keeps pointing at the join node. This allows + * lookups and updaters to determine if they should see a link between + * the tail L and F when searching for nodes originally in psrc_head + * via pdest_head. If they reach the tail of pdest_head without + * encountering any nodes of psrc_head, either there were no nodes + * in psrc_head to begin with or the link between L and F did not + * yet propagate to their cpus. If psrc_head was empty, it remains + * NULL. Otherwise psrc_head points to a join node (it will not be + * unlinked until table resize completes) and updaters/lookups + * should issue a read_barrier() to make the link [L]->[JN] visible. + * + * 0) ,-- split_hash, first hash of the src bucket + * v + * [dest_head | N]-> .. -> [L] + * [src_head | N]--> [F] -> .. + * ^ + * ` split_hash, first hash of the src bucket + * + * 1) ,-- split_hash + * v + * [dest_head | N]-> .. -> [L] + * [src_head | C]--> [F] -> .. + * + * 2) ,-- split_hash + * v + * [dest_head | N]-> .. -> [L] + * [src_head | C]--> [JN] -> .. + * + * 3) ,-- split_hash + * v + * [dest_head | N]-> .. -> [L] --+ + * v + * [src_head | C]-------------> [JN] -> .. + * + * 4) ,-- split_hash + * v + * [dest_head | N]-> .. -> [L] --+ + * v + * [src_head | Inv]-----------> [JN] -> .. + */ + + rcu_read_lock(); + + /* Mark src_head immutable - signals updaters that bucket join started. */ + mark_const(psrc_head); + cas_order_barrier(); + + cht_link_t *join_node = get_next(*psrc_head); + + if (join_node != &sentinel) { + mark_join_node(join_node); + cas_order_barrier(); + + link_to_join_node(h, pdest_head, join_node, split_hash); + cas_order_barrier(); + } + + marked_ptr_t ret = + cas_link(psrc_head, join_node, N_CONST, join_node, N_INVALID); + ASSERT(ret == make_link(join_node, N_CONST) || (N_INVALID == get_mark(ret))); + cas_order_barrier(); + + rcu_read_unlock(); +} + +static void link_to_join_node(cht_t *h, marked_ptr_t *pdest_head, + cht_link_t *join_node, size_t split_hash) +{ + bool done; + do { + wnd_t wnd = { + .ppred = pdest_head, + .cur = get_next(*pdest_head) + }; + + bool resizing = false; + + if (!find_wnd_and_gc(h, split_hash, WM_LEAVE_JOIN, &wnd, &resizing)) + continue; + + ASSERT(!resizing); + + if (wnd.cur != &sentinel) { + /* Must be from the new appended bucket. */ + ASSERT(split_hash <= node_hash(h, wnd.cur) + || h->invalid_hash == node_hash(h, wnd.cur)); + return; + } + + /* Reached the tail of pdest_head - link it to the join node. */ + marked_ptr_t ret = + cas_link(wnd.ppred, &sentinel, N_NORMAL, join_node, N_NORMAL); + + done = (ret == make_link(&sentinel, N_NORMAL)); + } while (!done); +} + +static void free_later(cht_t *h, cht_link_t *item) +{ + ASSERT(item != &sentinel); + + /* + * remove_callback only works as rcu_func_t because rcu_link is the first + * field in cht_link_t. + */ + rcu_call(&item->rcu_link, (rcu_func_t)h->op->remove_callback); + + item_removed(h); +} + +static inline void item_removed(cht_t *h) +{ + size_t items = (size_t) atomic_predec(&h->item_cnt); + size_t bucket_cnt = (1 << h->b->order); + + bool need_shrink = (items == h->max_load * bucket_cnt / 4); + bool missed_shrink = (items == h->max_load * bucket_cnt / 8); + + if ((need_shrink || missed_shrink) && h->b->order > h->min_order) { + atomic_count_t resize_reqs = atomic_preinc(&h->resize_reqs); + /* The first resize request. Start the resizer. */ + if (1 == resize_reqs) { + workq_global_enqueue_noblock(&h->resize_work, resize_table); + } + } +} + +static inline void item_inserted(cht_t *h) +{ + size_t items = (size_t) atomic_preinc(&h->item_cnt); + size_t bucket_cnt = (1 << h->b->order); + + bool need_grow = (items == h->max_load * bucket_cnt); + bool missed_grow = (items == 2 * h->max_load * bucket_cnt); + + if ((need_grow || missed_grow) && h->b->order < CHT_MAX_ORDER) { + atomic_count_t resize_reqs = atomic_preinc(&h->resize_reqs); + /* The first resize request. Start the resizer. */ + if (1 == resize_reqs) { + workq_global_enqueue_noblock(&h->resize_work, resize_table); + } + } +} + +static void resize_table(work_t *arg) +{ + cht_t *h = member_to_inst(arg, cht_t, resize_work); + +#ifdef CONFIG_DEBUG + ASSERT(h->b); + /* Make resize_reqs visible. */ + read_barrier(); + ASSERT(0 < atomic_get(&h->resize_reqs)); +#endif + + bool done; + do { + /* Load the most recent h->item_cnt. */ + read_barrier(); + size_t cur_items = (size_t) atomic_get(&h->item_cnt); + size_t bucket_cnt = (1 << h->b->order); + size_t max_items = h->max_load * bucket_cnt; + + if (cur_items >= max_items && h->b->order < CHT_MAX_ORDER) { + grow_table(h); + } else if (cur_items <= max_items / 4 && h->b->order > h->min_order) { + shrink_table(h); + } else { + /* Table is just the right size. */ + atomic_count_t reqs = atomic_predec(&h->resize_reqs); + done = (reqs == 0); + } + } while (!done); +} + +static void grow_table(cht_t *h) +{ + if (h->b->order >= CHT_MAX_ORDER) + return; + + h->new_b = alloc_buckets(h->b->order + 1, true); + + /* Failed to alloc a new table - try next time the resizer is run. */ + if (!h->new_b) + return; + + /* Wait for all readers and updaters to see the initialized new table. */ + rcu_synchronize(); + size_t old_bucket_cnt = (1 << h->b->order); + + /* + * Give updaters a chance to help out with the resize. Do the minimum + * work needed to announce a resize is in progress, ie start moving heads. + */ + for (size_t idx = 0; idx < old_bucket_cnt; ++idx) { + start_head_move(&h->b->head[idx]); + } + + /* Order start_head_move() wrt complete_head_move(). */ + cas_order_barrier(); + + /* Complete moving heads and split any buckets not yet split by updaters. */ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + marked_ptr_t *move_dest_head = &h->new_b->head[grow_idx(old_idx)]; + marked_ptr_t *move_src_head = &h->b->head[old_idx]; + + /* Head move not yet completed. */ + if (N_INVALID != get_mark(*move_src_head)) { + complete_head_move(move_src_head, move_dest_head); + } + + size_t split_idx = grow_to_split_idx(old_idx); + size_t split_hash = calc_split_hash(split_idx, h->new_b->order); + marked_ptr_t *split_dest_head = &h->new_b->head[split_idx]; + + split_bucket(h, move_dest_head, split_dest_head, split_hash); + } + + /* + * Wait for all updaters to notice the new heads. Once everyone sees + * the invalid old bucket heads they will know a resize is in progress + * and updaters will modify the correct new buckets. + */ + rcu_synchronize(); + + /* Clear the JOIN_FOLLOWS mark and remove the link between the split buckets.*/ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + size_t new_idx = grow_idx(old_idx); + + cleanup_join_follows(h, &h->new_b->head[new_idx]); + } + + /* + * Wait for everyone to notice that buckets were split, ie link connecting + * the join follows and join node has been cut. + */ + rcu_synchronize(); + + /* Clear the JOIN mark and GC any deleted join nodes. */ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + size_t new_idx = grow_to_split_idx(old_idx); + + cleanup_join_node(h, &h->new_b->head[new_idx]); + } + + /* Wait for everyone to see that the table is clear of any resize marks. */ + rcu_synchronize(); + + cht_buckets_t *old_b = h->b; + rcu_assign(h->b, h->new_b); + + /* Wait for everyone to start using the new table. */ + rcu_synchronize(); + + free(old_b); + + /* Not needed; just for increased readability. */ + h->new_b = 0; +} + +static void shrink_table(cht_t *h) +{ + if (h->b->order <= h->min_order) + return; + + h->new_b = alloc_buckets(h->b->order - 1, true); + + /* Failed to alloc a new table - try next time the resizer is run. */ + if (!h->new_b) + return; + + /* Wait for all readers and updaters to see the initialized new table. */ + rcu_synchronize(); + + size_t old_bucket_cnt = (1 << h->b->order); + + /* + * Give updaters a chance to help out with the resize. Do the minimum + * work needed to announce a resize is in progress, ie start moving heads. + */ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + size_t new_idx = shrink_idx(old_idx); + + /* This bucket should be moved. */ + if (grow_idx(new_idx) == old_idx) { + start_head_move(&h->b->head[old_idx]); + } else { + /* This bucket should join the moved bucket once the move is done.*/ + } + } + + /* Order start_head_move() wrt to complete_head_move(). */ + cas_order_barrier(); + + /* Complete moving heads and join buckets with the moved buckets. */ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + size_t new_idx = shrink_idx(old_idx); + size_t move_src_idx = grow_idx(new_idx); + + /* This bucket should be moved. */ + if (move_src_idx == old_idx) { + /* Head move not yet completed. */ + if (N_INVALID != get_mark(h->b->head[old_idx])) { + complete_head_move(&h->b->head[old_idx], &h->new_b->head[new_idx]); + } + } else { + /* This bucket should join the moved bucket. */ + size_t split_hash = calc_split_hash(old_idx, h->b->order); + join_buckets(h, &h->b->head[old_idx], &h->new_b->head[new_idx], + split_hash); + } + } + + /* + * Wait for all updaters to notice the new heads. Once everyone sees + * the invalid old bucket heads they will know a resize is in progress + * and updaters will modify the correct new buckets. + */ + rcu_synchronize(); + + /* Let everyone know joins are complete and fully visible. */ + for (size_t old_idx = 0; old_idx < old_bucket_cnt; ++old_idx) { + size_t move_src_idx = grow_idx(shrink_idx(old_idx)); + + /* Set the invalid joinee head to NULL. */ + if (old_idx != move_src_idx) { + ASSERT(N_INVALID == get_mark(h->b->head[old_idx])); + + if (&sentinel != get_next(h->b->head[old_idx])) + h->b->head[old_idx] = make_link(&sentinel, N_INVALID); + } + } + + /* todo comment join node vs reset joinee head*/ + rcu_synchronize(); + + size_t new_bucket_cnt = (1 << h->new_b->order); + + /* Clear the JOIN mark and GC any deleted join nodes. */ + for (size_t new_idx = 0; new_idx < new_bucket_cnt; ++new_idx) { + cleanup_join_node(h, &h->new_b->head[new_idx]); + } + + /* Wait for everyone to see that the table is clear of any resize marks. */ + rcu_synchronize(); + + cht_buckets_t *old_b = h->b; + rcu_assign(h->b, h->new_b); + + /* Wait for everyone to start using the new table. */ + rcu_synchronize(); + + free(old_b); + + /* Not needed; just for increased readability. */ + h->new_b = 0; +} + +static void cleanup_join_node(cht_t *h, marked_ptr_t *new_head) +{ + rcu_read_lock(); + + cht_link_t *cur = get_next(*new_head); + + while (cur != &sentinel) { + /* Clear the join node's JN mark - even if it is marked as deleted. */ + if (N_JOIN & get_mark(cur->link)) { + clear_join_and_gc(h, cur, new_head); + break; + } + + cur = get_next(cur->link); + } + + rcu_read_unlock(); +} + +static void clear_join_and_gc(cht_t *h, cht_link_t *join_node, + marked_ptr_t *new_head) +{ + ASSERT(join_node != &sentinel); + ASSERT(join_node && (N_JOIN & get_mark(join_node->link))); + + bool done; + + /* Clear the JN mark. */ + do { + marked_ptr_t jn_link = join_node->link; + cht_link_t *next = get_next(jn_link); + /* Clear the JOIN mark but keep the DEL mark if present. */ + mark_t cleared_mark = get_mark(jn_link) & N_DELETED; + + marked_ptr_t ret = + _cas_link(&join_node->link, jn_link, make_link(next, cleared_mark)); + + /* Done if the mark was cleared. Retry if a new node was inserted. */ + done = (ret == jn_link); + ASSERT(ret == jn_link || (get_mark(ret) & N_JOIN)); + } while (!done); + + if (!(N_DELETED & get_mark(join_node->link))) + return; + + /* The join node had been marked as deleted - GC it. */ + + /* Clear the JOIN mark before trying to unlink the deleted join node.*/ + cas_order_barrier(); + + size_t jn_hash = node_hash(h, join_node); + do { + bool resizing = false; + + wnd_t wnd = { + .ppred = new_head, + .cur = get_next(*new_head) + }; + + done = find_wnd_and_gc_pred(h, jn_hash, WM_NORMAL, same_node_pred, + join_node, &wnd, &resizing); + + ASSERT(!resizing); + } while (!done); +} + +static void cleanup_join_follows(cht_t *h, marked_ptr_t *new_head) +{ + ASSERT(new_head); + + rcu_read_lock(); + + wnd_t wnd = { + .ppred = 0, + .cur = 0 + }; + marked_ptr_t *cur_link = new_head; + + /* + * Find the non-deleted node with a JF mark and clear the JF mark. + * The JF node may be deleted and/or the mark moved to its neighbors + * at any time. Therefore, we GC deleted nodes until we find the JF + * node in order to remove stale/deleted JF nodes left behind eg by + * delayed threads that did not yet get a chance to unlink the deleted + * JF node and move its mark. + * + * Note that the head may be marked JF (but never DELETED). + */ + while (true) { + bool is_jf_node = N_JOIN_FOLLOWS & get_mark(*cur_link); + + /* GC any deleted nodes on the way - even deleted JOIN_FOLLOWS. */ + if (N_DELETED & get_mark(*cur_link)) { + ASSERT(cur_link != new_head); + ASSERT(wnd.ppred && wnd.cur && wnd.cur != &sentinel); + ASSERT(cur_link == &wnd.cur->link); + + bool dummy; + bool deleted = gc_deleted_node(h, WM_MOVE_JOIN_FOLLOWS, &wnd, &dummy); + + /* Failed to GC or collected a deleted JOIN_FOLLOWS. */ + if (!deleted || is_jf_node) { + /* Retry from the head of the bucket. */ + cur_link = new_head; + continue; + } + } else { + /* Found a non-deleted JF. Clear its JF mark. */ + if (is_jf_node) { + cht_link_t *next = get_next(*cur_link); + marked_ptr_t ret = + cas_link(cur_link, next, N_JOIN_FOLLOWS, &sentinel, N_NORMAL); + + ASSERT(next == &sentinel + || ((N_JOIN | N_JOIN_FOLLOWS) & get_mark(ret))); + + /* Successfully cleared the JF mark of a non-deleted node. */ + if (ret == make_link(next, N_JOIN_FOLLOWS)) { + break; + } else { + /* + * The JF node had been deleted or a new node inserted + * right after it. Retry from the head. + */ + cur_link = new_head; + continue; + } + } else { + wnd.ppred = cur_link; + wnd.cur = get_next(*cur_link); + } + } + + /* We must encounter a JF node before we reach the end of the bucket. */ + ASSERT(wnd.cur && wnd.cur != &sentinel); + cur_link = &wnd.cur->link; + } + + rcu_read_unlock(); +} + + +static inline size_t calc_split_hash(size_t split_idx, size_t order) +{ + ASSERT(1 <= order && order <= 8 * sizeof(size_t)); + return split_idx << (8 * sizeof(size_t) - order); +} + +static inline size_t calc_bucket_idx(size_t hash, size_t order) +{ + ASSERT(1 <= order && order <= 8 * sizeof(size_t)); + return hash >> (8 * sizeof(size_t) - order); +} + +static inline size_t grow_to_split_idx(size_t old_idx) +{ + return grow_idx(old_idx) | 1; +} + +static inline size_t grow_idx(size_t idx) +{ + return idx << 1; +} + +static inline size_t shrink_idx(size_t idx) +{ + return idx >> 1; +} + +static inline size_t calc_key_hash(cht_t *h, void *key) +{ + /* Mimick calc_node_hash. */ + return hash_mix(h->op->key_hash(key)) & ~1U; +} + +static inline size_t node_hash(cht_t *h, const cht_link_t *item) +{ + ASSERT(item->hash == h->invalid_hash + || item->hash == sentinel.hash + || item->hash == calc_node_hash(h, item)); + + return item->hash; +} + +static inline size_t calc_node_hash(cht_t *h, const cht_link_t *item) +{ + ASSERT(item != &sentinel); + /* + * Clear the lowest order bit in order for sentinel's node hash + * to be the greatest possible. + */ + return hash_mix(h->op->hash(item)) & ~1U; +} + +static inline void memoize_node_hash(cht_t *h, cht_link_t *item) +{ + item->hash = calc_node_hash(h, item); +} + +static inline marked_ptr_t make_link(const cht_link_t *next, mark_t mark) +{ + marked_ptr_t ptr = (marked_ptr_t) next; + + ASSERT(!(ptr & N_MARK_MASK)); + ASSERT(!((unsigned)mark & ~N_MARK_MASK)); + + return ptr | mark; +} + + +static inline cht_link_t * get_next(marked_ptr_t link) +{ + return (cht_link_t*)(link & ~N_MARK_MASK); +} + + +static inline mark_t get_mark(marked_ptr_t link) +{ + return (mark_t)(link & N_MARK_MASK); +} + + +static inline void next_wnd(wnd_t *wnd) +{ + ASSERT(wnd); + ASSERT(wnd->cur); + + wnd->last = wnd->cur; + wnd->ppred = &wnd->cur->link; + wnd->cur = get_next(wnd->cur->link); +} + + +static bool same_node_pred(void *node, const cht_link_t *item2) +{ + const cht_link_t *item1 = (const cht_link_t*) node; + return item1 == item2; +} + +static inline marked_ptr_t cas_link(marked_ptr_t *link, const cht_link_t *cur_next, + mark_t cur_mark, const cht_link_t *new_next, mark_t new_mark) +{ + return _cas_link(link, make_link(cur_next, cur_mark), + make_link(new_next, new_mark)); +} + +static inline marked_ptr_t _cas_link(marked_ptr_t *link, marked_ptr_t cur, + marked_ptr_t new) +{ + ASSERT(link != &sentinel.link); + /* + * cas(x) on the same location x on one cpu must be ordered, but do not + * have to be ordered wrt to other cas(y) to a different location y + * on the same cpu. + * + * cas(x) must act as a write barrier on x, ie if cas(x) succeeds + * and is observed by another cpu, then all cpus must be able to + * make the effects of cas(x) visible just by issuing a load barrier. + * For example: + * cpu1 cpu2 cpu3 + * cas(x, 0 -> 1), succeeds + * cas(x, 0 -> 1), fails + * MB + * y = 7 + * sees y == 7 + * loadMB must be enough to make cas(x) on cpu3 visible to cpu1, ie x == 1. + * + * If cas() did not work this way: + * - our head move protocol would not be correct. + * - freeing an item linked to a moved head after another item was + * inserted in front of it, would require more than one grace period. + */ + void *ret = atomic_cas_ptr((void**)link, (void *)cur, (void *)new); + return (marked_ptr_t) ret; +} + +static inline void cas_order_barrier(void) +{ + /* Make sure CAS to different memory locations are ordered. */ + write_barrier(); +} + + +/** @} + */ Index: kernel/generic/src/adt/list.c =================================================================== --- kernel/generic/src/adt/list.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/adt/list.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -68,24 +68,26 @@ } -/** Concatenate two lists - * - * Concatenate lists @a list1 and @a list2, producing a single - * list @a list1 containing items from both (in @a list1, @a list2 - * order) and empty list @a list2. - * - * @param list1 First list and concatenated output - * @param list2 Second list and empty output. - * +/** Moves items of one list into another after the specified item. + * + * Inserts all items of @a list after item at @a pos in another list. + * Both lists may be empty. + * + * @param list Source list to move after pos. Empty afterwards. + * @param pos Source items will be placed after this item. */ -void list_concat(list_t *list1, list_t *list2) +void list_splice(list_t *list, link_t *pos) { - if (list_empty(list2)) + if (list_empty(list)) return; - - list2->head.next->prev = list1->head.prev; - list2->head.prev->next = &list1->head; - list1->head.prev->next = list2->head.next; - list1->head.prev = list2->head.prev; - list_initialize(list2); + + /* Attach list to destination. */ + list->head.next->prev = pos; + list->head.prev->next = pos->next; + + /* Link destination list to the added list. */ + pos->next->prev = list->head.prev; + pos->next = list->head.next; + + list_initialize(list); } Index: kernel/generic/src/console/chardev.c =================================================================== --- kernel/generic/src/console/chardev.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/console/chardev.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -39,5 +39,5 @@ #include #include -#include +#include /** Initialize input character device. Index: kernel/generic/src/console/cmd.c =================================================================== --- kernel/generic/src/console/cmd.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/console/cmd.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -68,4 +68,6 @@ #include #include +#include +#include #include @@ -449,4 +451,22 @@ }; +/* Data and methods for the 'workq' command */ +static int cmd_workq(cmd_arg_t *argv); +static cmd_info_t workq_info = { + .name = "workq", + .description = "Show global workq information.", + .func = cmd_workq, + .argc = 0 +}; + +/* Data and methods for the 'workq' command */ +static int cmd_rcu(cmd_arg_t *argv); +static cmd_info_t rcu_info = { + .name = "rcu", + .description = "Show RCU run-time statistics.", + .func = cmd_rcu, + .argc = 0 +}; + /* Data and methods for 'ipc' command */ static int cmd_ipc(cmd_arg_t *argv); @@ -512,4 +532,5 @@ &physmem_info, &reboot_info, + &rcu_info, &sched_info, &set4_info, @@ -522,4 +543,5 @@ &uptime_info, &version_info, + &workq_info, &zones_info, &zone_info, @@ -1015,4 +1037,28 @@ } +/** Prints information about the global work queue. + * + * @param argv Ignores + * + * @return Always 1 + */ +int cmd_workq(cmd_arg_t *argv) +{ + workq_global_print_info(); + return 1; +} + +/** Prints RCU statistics. + * + * @param argv Ignores + * + * @return Always 1 + */ +int cmd_rcu(cmd_arg_t *argv) +{ + rcu_print_stat(); + return 1; +} + /** Command for listing memory zones * Index: kernel/generic/src/console/console.c =================================================================== --- kernel/generic/src/console/console.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/console/console.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -52,4 +52,6 @@ #include #include +#include /* SIZE2FRAMES */ +#include /* malloc */ #define KLOG_PAGES 8 Index: kernel/generic/src/console/kconsole.c =================================================================== --- kernel/generic/src/console/kconsole.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/console/kconsole.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -60,4 +60,5 @@ #include #include +#include /** Simple kernel console. Index: kernel/generic/src/cpu/cpu.c =================================================================== --- kernel/generic/src/cpu/cpu.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/cpu/cpu.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -50,4 +50,5 @@ #include #include +#include cpu_t *cpus; @@ -102,4 +103,5 @@ cpu_identify(); cpu_arch_init(); + rcu_cpu_init(); } Index: kernel/generic/src/cpu/cpu_mask.c =================================================================== --- kernel/generic/src/cpu/cpu_mask.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/src/cpu/cpu_mask.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ + +/** + * @file + * @brief CPU mask manipulation functions. + */ +#include +#include +#include + +static const size_t word_size = sizeof(unsigned int); +static const size_t word_bit_cnt = 8 * sizeof(unsigned int); + +/** Returns the size of cpu_mask_t for the detected number of cpus in bytes. */ +size_t cpu_mask_size(void) +{ + size_t word_cnt = (config.cpu_count + word_bit_cnt - 1) / word_bit_cnt; + return word_cnt * word_size; +} + +/** Add first cpu_cnt cpus to the mask, ie sets the first cpu_cnt bits. */ +static void cpu_mask_count(cpu_mask_t *cpus, size_t cpu_cnt) +{ + ASSERT(NULL != cpus); + ASSERT(cpu_cnt <= config.cpu_count); + + for (size_t active_word = 0; + (active_word + 1) * word_bit_cnt <= cpu_cnt; + ++active_word) { + /* Set all bits in the cell/word. */ + cpus->mask[active_word] = -1; + } + + size_t remaining_bits = (cpu_cnt % word_bit_cnt); + if (0 < remaining_bits) { + /* Set lower remaining_bits of the last word. */ + cpus->mask[cpu_cnt / word_bit_cnt] = (1 << remaining_bits) - 1; + } +} + +/** Sets bits corresponding to the active cpus, ie the first + * config.cpu_active cpus. + */ +void cpu_mask_active(cpu_mask_t *cpus) +{ + cpu_mask_none(cpus); + cpu_mask_count(cpus, config.cpu_active); +} + +/** Sets bits for all cpus of the mask. */ +void cpu_mask_all(cpu_mask_t *cpus) +{ + cpu_mask_count(cpus, config.cpu_count); +} + +/** Resets/removes all bits. */ +void cpu_mask_none(cpu_mask_t *cpus) +{ + ASSERT(cpus); + + size_t word_cnt = cpu_mask_size() / word_size; + + for (size_t word = 0; word < word_cnt; ++word) { + cpus->mask[word] = 0; + } +} + +/** Sets the bit corresponding to cpu_id to true. */ +void cpu_mask_set(cpu_mask_t *cpus, unsigned int cpu_id) +{ + size_t word = cpu_id / word_bit_cnt; + size_t word_pos = cpu_id % word_bit_cnt; + + cpus->mask[word] |= (1U << word_pos); +} + +/** Resets the bit corresponding to cpu_id to false. */ +void cpu_mask_reset(cpu_mask_t *cpus, unsigned int cpu_id) +{ + size_t word = cpu_id / word_bit_cnt; + size_t word_pos = cpu_id % word_bit_cnt; + + cpus->mask[word] &= ~(1U << word_pos); +} + +/** Returns true if the bit corresponding to cpu_id is set. */ +bool cpu_mask_is_set(cpu_mask_t *cpus, unsigned int cpu_id) +{ + size_t word = cpu_id / word_bit_cnt; + size_t word_pos = cpu_id % word_bit_cnt; + + return 0 != (cpus->mask[word] & (1U << word_pos)); +} + +/** Returns true if no bits are set. */ +bool cpu_mask_is_none(cpu_mask_t *cpus) +{ + size_t word_cnt = cpu_mask_size() / word_size; + + for (size_t word = 0; word < word_cnt; ++word) { + if (cpus->mask[word]) + return false; + } + + return true; +} + +/** @} + */ Index: kernel/generic/src/debug/panic.c =================================================================== --- kernel/generic/src/debug/panic.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/debug/panic.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -97,5 +97,5 @@ if (THE != NULL) { printf("pe=%" PRIun " thr=%p task=%p cpu=%p as=%p" - " magic=%#" PRIx32 "\n", THE->preemption_disabled, + " magic=%#" PRIx32 "\n", THE->preemption, THE->thread, THE->task, THE->cpu, THE->as, THE->magic); } else Index: kernel/generic/src/interrupt/interrupt.c =================================================================== --- kernel/generic/src/interrupt/interrupt.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/interrupt/interrupt.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -111,13 +111,11 @@ } - /* Account CPU usage if it has waked up from sleep */ - if (CPU) { + /* Account CPU usage if it woke up from sleep */ + if (CPU && CPU->idle) { irq_spinlock_lock(&CPU->lock, false); - if (CPU->idle) { - uint64_t now = get_cycle(); - CPU->idle_cycles += now - CPU->last_cycle; - CPU->last_cycle = now; - CPU->idle = false; - } + uint64_t now = get_cycle(); + CPU->idle_cycles += now - CPU->last_cycle; + CPU->last_cycle = now; + CPU->idle = false; irq_spinlock_unlock(&CPU->lock, false); } Index: kernel/generic/src/ipc/kbox.c =================================================================== --- kernel/generic/src/ipc/kbox.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/ipc/kbox.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -44,4 +44,5 @@ #include #include +#include void ipc_kbox_cleanup(void) Index: kernel/generic/src/lib/str.c =================================================================== --- kernel/generic/src/lib/str.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/lib/str.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -111,4 +111,5 @@ #include #include +#include /** Check the condition if wchar_t is signed */ @@ -567,4 +568,5 @@ /* There must be space for a null terminator in the buffer. */ ASSERT(size > 0); + ASSERT(src != NULL); size_t src_off = 0; Index: kernel/generic/src/main/kinit.c =================================================================== --- kernel/generic/src/main/kinit.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/main/kinit.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -77,4 +77,6 @@ #include #include +#include +#include #define ALIVE_CHARS 4 @@ -103,6 +105,14 @@ */ thread_detach(THREAD); - + interrupts_disable(); + + /* Start processing RCU callbacks. RCU is fully functional afterwards. */ + rcu_kinit_init(); + + /* + * Start processing work queue items. Some may have been queued during boot. + */ + workq_global_worker_init(); #ifdef CONFIG_SMP Index: kernel/generic/src/main/main.c =================================================================== --- kernel/generic/src/main/main.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/main/main.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -75,4 +75,6 @@ #include #include +#include +#include #include #include @@ -244,6 +246,9 @@ cpu_init(); - calibrate_delay_loop(); + arch_post_cpu_init(); + + smp_call_init(); + workq_global_init(); clock_counter_init(); timeout_init(); @@ -347,4 +352,6 @@ void main_ap_separated_stack(void) { + smp_call_init(); + /* * Configure timeouts for this cpu. Index: kernel/generic/src/main/shutdown.c =================================================================== --- kernel/generic/src/main/shutdown.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/main/shutdown.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -37,4 +37,5 @@ #include +#include #include #include Index: kernel/generic/src/mm/frame.c =================================================================== --- kernel/generic/src/mm/frame.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/mm/frame.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -61,4 +61,5 @@ #include #include +#include /* THREAD */ zones_t zones; Index: kernel/generic/src/mm/km.c =================================================================== --- kernel/generic/src/mm/km.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/mm/km.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -49,4 +49,5 @@ #include #include +#include static ra_arena_t *km_ni_arena; Index: kernel/generic/src/mm/slab.c =================================================================== --- kernel/generic/src/mm/slab.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/mm/slab.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -114,4 +114,5 @@ #include #include +#include IRQ_SPINLOCK_STATIC_INITIALIZE(slab_cache_lock); Index: kernel/generic/src/preempt/preemption.c =================================================================== --- kernel/generic/src/preempt/preemption.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/preempt/preemption.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -37,23 +37,5 @@ #include -#include -#include -#include -#include -/** Increment preemption disabled counter. */ -void preemption_disable(void) -{ - THE->preemption_disabled++; - memory_barrier(); -} - -/** Decrement preemption disabled counter. */ -void preemption_enable(void) -{ - ASSERT(PREEMPTION_DISABLED); - memory_barrier(); - THE->preemption_disabled--; -} /** @} Index: kernel/generic/src/proc/scheduler.c =================================================================== --- kernel/generic/src/proc/scheduler.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/proc/scheduler.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -52,4 +52,6 @@ #include #include +#include +#include #include #include @@ -63,4 +65,5 @@ #include #include +#include static void scheduler_separated_stack(void); @@ -86,4 +89,5 @@ { before_thread_runs_arch(); + rcu_before_thread_runs(); #ifdef CONFIG_FPU_LAZY @@ -126,4 +130,6 @@ static void after_thread_ran(void) { + workq_after_thread_ran(); + rcu_after_thread_ran(); after_thread_ran_arch(); } @@ -218,4 +224,6 @@ goto loop; } + + ASSERT(!CPU->idle); unsigned int i; @@ -397,4 +405,5 @@ ASSERT((!THREAD) || (irq_spinlock_locked(&THREAD->lock))); ASSERT(CPU != NULL); + ASSERT(interrupts_disabled()); /* @@ -420,4 +429,5 @@ case Exiting: + rcu_thread_exiting(); repeat: if (THREAD->detached) { Index: kernel/generic/src/proc/the.c =================================================================== --- kernel/generic/src/proc/the.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/proc/the.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -43,4 +43,5 @@ #include +#include /** Initialize THE structure @@ -53,5 +54,5 @@ void the_initialize(the_t *the) { - the->preemption_disabled = 0; + the->preemption = 0; the->cpu = NULL; the->thread = NULL; @@ -59,4 +60,7 @@ the->as = NULL; the->magic = MAGIC; +#ifdef RCU_PREEMPT_A + the->rcu_nesting = 0; +#endif } Index: kernel/generic/src/proc/thread.c =================================================================== --- kernel/generic/src/proc/thread.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/proc/thread.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -46,4 +46,6 @@ #include #include +#include +#include #include #include @@ -260,4 +262,11 @@ } +/** Invoked right before thread_ready() readies the thread. thread is locked. */ +static void before_thread_is_ready(thread_t *thread) +{ + ASSERT(irq_spinlock_locked(&thread->lock)); + workq_before_thread_is_ready(thread); +} + /** Make thread ready * @@ -272,13 +281,20 @@ ASSERT(thread->state != Ready); + + before_thread_is_ready(thread); int i = (thread->priority < RQ_COUNT - 1) ? ++thread->priority : thread->priority; - - cpu_t *cpu; - if (thread->wired || thread->nomigrate || thread->fpu_context_engaged) { - ASSERT(thread->cpu != NULL); - cpu = thread->cpu; - } else + + /* Check that thread->cpu is set whenever it needs to be. */ + ASSERT(thread->cpu != NULL || + (!thread->wired && !thread->nomigrate && !thread->fpu_context_engaged)); + + /* + * Prefer to run on the same cpu as the last time. Used by wired + * threads as well as threads with disabled migration. + */ + cpu_t *cpu = thread->cpu; + if (cpu == NULL) cpu = CPU; @@ -374,4 +390,6 @@ thread->task = task; + thread->workq = NULL; + thread->fpu_context_exists = false; thread->fpu_context_engaged = false; @@ -388,4 +406,6 @@ /* Might depend on previous initialization */ thread_create_arch(thread); + + rcu_thread_init(thread); if ((flags & THREAD_FLAG_NOATTACH) != THREAD_FLAG_NOATTACH) @@ -518,4 +538,52 @@ /* Not reached */ while (true); +} + +/** Interrupts an existing thread so that it may exit as soon as possible. + * + * Threads that are blocked waiting for a synchronization primitive + * are woken up with a return code of ESYNCH_INTERRUPTED if the + * blocking call was interruptable. See waitq_sleep_timeout(). + * + * The caller must guarantee the thread object is valid during the entire + * function, eg by holding the threads_lock lock. + * + * Interrupted threads automatically exit when returning back to user space. + * + * @param thread A valid thread object. The caller must guarantee it + * will remain valid until thread_interrupt() exits. + */ +void thread_interrupt(thread_t *thread) +{ + ASSERT(thread != NULL); + + irq_spinlock_lock(&thread->lock, true); + + thread->interrupted = true; + bool sleeping = (thread->state == Sleeping); + + irq_spinlock_unlock(&thread->lock, true); + + if (sleeping) + waitq_interrupt_sleep(thread); +} + +/** Returns true if the thread was interrupted. + * + * @param thread A valid thread object. User must guarantee it will + * be alive during the entire call. + * @return true if the thread was already interrupted via thread_interrupt(). + */ +bool thread_interrupted(thread_t *thread) +{ + ASSERT(thread != NULL); + + bool interrupted; + + irq_spinlock_lock(&thread->lock, true); + interrupted = thread->interrupted; + irq_spinlock_unlock(&thread->lock, true); + + return interrupted; } Index: kernel/generic/src/smp/smp_call.c =================================================================== --- kernel/generic/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/src/smp/smp_call.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ + +/** + * @file + * @brief Facility to invoke functions on other cpus via IPIs. + */ + +#include +#include +#include /* interrupt_disable */ +#include +#include +#include +#include +#include + +static void call_start(smp_call_t *call_info, smp_call_func_t func, void *arg); +static void call_done(smp_call_t *call_info); +static void call_wait(smp_call_t *call_info); + + +/** Init smp_call() on the local cpu. */ +void smp_call_init(void) +{ + ASSERT(CPU); + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + + spinlock_initialize(&CPU->smp_calls_lock, "cpu[].smp_calls_lock"); + list_initialize(&CPU->smp_pending_calls); +} + +/** Invokes a function on a specific cpu and waits for it to complete. + * + * Calls @a func on the CPU denoted by its logical id @cpu_id . + * The function will execute with interrupts disabled. It should + * be a quick and simple function and must never block. + * + * If @a cpu_id is the local CPU, the function will be invoked + * directly. + * + * @param cpu_id Destination CPU's logical id (eg CPU->id) + * @param func Function to call. + * @param arg Argument to pass to the user supplied function @a func. + */ +void smp_call(unsigned int cpu_id, smp_call_func_t func, void *arg) +{ + smp_call_t call_info; + smp_call_async(cpu_id, func, arg, &call_info); + smp_call_wait(&call_info); +} + +/** Invokes a function on a specific cpu asynchronously. + * + * Calls @a func on the CPU denoted by its logical id @cpu_id . + * The function will execute with interrupts disabled. It should + * be a quick and simple function and must never block. + * + * Pass @a call_info to smp_call_wait() in order to wait for + * @a func to complete. + * + * @a call_info must be valid until @a func returns. + * + * If @a cpu_id is the local CPU, the function will be invoked + * directly. If the destination cpu id @a cpu_id is invalid + * or denotes an inactive cpu, the call is discarded immediately. + * + * Interrupts must be enabled. Otherwise you run the risk + * of a deadlock. + * + * @param cpu_id Destination CPU's logical id (eg CPU->id). + * @param func Function to call. + * @param arg Argument to pass to the user supplied function @a func. + * @param call_info Use it to wait for the function to complete. Must + * be valid until the function completes. + */ +void smp_call_async(unsigned int cpu_id, smp_call_func_t func, void *arg, + smp_call_t *call_info) +{ + /* todo: doc deadlock */ + ASSERT(!interrupts_disabled()); + ASSERT(call_info != NULL); + + /* Discard invalid calls. */ + if (config.cpu_count <= cpu_id || !cpus[cpu_id].active) { + call_start(call_info, func, arg); + call_done(call_info); + return; + } + + /* Protect cpu->id against migration. */ + preemption_disable(); + + call_start(call_info, func, arg); + + if (cpu_id != CPU->id) { +#ifdef CONFIG_SMP + spinlock_lock(&cpus[cpu_id].smp_calls_lock); + list_append(&call_info->calls_link, &cpus[cpu_id].smp_pending_calls); + spinlock_unlock(&cpus[cpu_id].smp_calls_lock); + + /* + * If a platform supports SMP it must implement arch_smp_call_ipi(). + * It should issue an IPI an cpu_id and invoke smp_call_ipi_recv() + * on cpu_id in turn. + * + * Do not implement as just an empty dummy function. Instead + * consider providing a full implementation or at least a version + * that panics if invoked. Note that smp_call_async() never + * calls arch_smp_call_ipi() on uniprocessors even if CONFIG_SMP. + */ + arch_smp_call_ipi(cpu_id); +#endif + } else { + /* Invoke local smp calls in place. */ + ipl_t ipl = interrupts_disable(); + func(arg); + interrupts_restore(ipl); + + call_done(call_info); + } + + preemption_enable(); +} + +/** Waits for a function invoked on another CPU asynchronously to complete. + * + * Does not sleep but rather spins. + * + * Example usage: + * @code + * void hello(void *p) { + * puts((char*)p); + * } + * + * smp_call_t call_info; + * smp_call_async(cpus[2].id, hello, "hi!\n", &call_info); + * // Do some work. In the meantime, hello() is executed on cpu2. + * smp_call_wait(&call_info); + * @endcode + * + * @param call_info Initialized by smp_call_async(). + */ +void smp_call_wait(smp_call_t *call_info) +{ + call_wait(call_info); +} + +#ifdef CONFIG_SMP + +/** Architecture independent smp call IPI handler. + * + * Interrupts must be disabled. Tolerates spurious calls. + */ +void smp_call_ipi_recv(void) +{ + ASSERT(interrupts_disabled()); + ASSERT(CPU); + + list_t calls_list; + list_initialize(&calls_list); + + spinlock_lock(&CPU->smp_calls_lock); + list_concat(&calls_list, &CPU->smp_pending_calls); + spinlock_unlock(&CPU->smp_calls_lock); + + /* Walk the list manually, so that we can safely remove list items. */ + for (link_t *cur = calls_list.head.next, *next = cur->next; + !list_empty(&calls_list); cur = next, next = cur->next) { + + smp_call_t *call_info = list_get_instance(cur, smp_call_t, calls_link); + list_remove(cur); + + call_info->func(call_info->arg); + call_done(call_info); + } +} + +#endif /* CONFIG_SMP */ + +static void call_start(smp_call_t *call_info, smp_call_func_t func, void *arg) +{ + link_initialize(&call_info->calls_link); + call_info->func = func; + call_info->arg = arg; + + /* + * We can't use standard spinlocks here because we want to lock + * the structure on one cpu and unlock it on another (without + * messing up the preemption count). + */ + atomic_set(&call_info->pending, 1); + + /* Let initialization complete before continuing. */ + memory_barrier(); +} + +static void call_done(smp_call_t *call_info) +{ + /* + * Separate memory accesses of the called function from the + * announcement of its completion. + */ + memory_barrier(); + atomic_set(&call_info->pending, 0); +} + +static void call_wait(smp_call_t *call_info) +{ + do { + /* + * Ensure memory accesses following call_wait() are ordered + * after completion of the called function on another cpu. + * Also, speed up loading of call_info->pending. + */ + memory_barrier(); + } while (atomic_get(&call_info->pending)); +} + + +/** @} + */ Index: kernel/generic/src/synch/condvar.c =================================================================== --- kernel/generic/src/synch/condvar.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/synch/condvar.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -38,4 +38,5 @@ #include #include +#include #include #include @@ -101,4 +102,85 @@ } +/** Wait for the condition to become true with a locked spinlock. + * + * The function is not aware of irq_spinlock. Therefore do not even + * try passing irq_spinlock_t to it. Use _condvar_wait_timeout_irq_spinlock() + * instead. + * + * @param cv Condition variable. + * @param lock Locked spinlock. + * @param usec Timeout value in microseconds. + * @param flags Select mode of operation. + * + * For exact description of meaning of possible combinations of usec and flags, + * see comment for waitq_sleep_timeout(). Note that when + * SYNCH_FLAGS_NON_BLOCKING is specified here, ESYNCH_WOULD_BLOCK is always + * returned. + * + * @return See comment for waitq_sleep_timeout(). + */ +int _condvar_wait_timeout_spinlock(condvar_t *cv, spinlock_t *lock, + uint32_t usec, int flags) +{ + int rc; + ipl_t ipl; + + ipl = waitq_sleep_prepare(&cv->wq); + + spinlock_unlock(lock); + + cv->wq.missed_wakeups = 0; /* Enforce blocking. */ + rc = waitq_sleep_timeout_unsafe(&cv->wq, usec, flags); + + waitq_sleep_finish(&cv->wq, rc, ipl); + + spinlock_lock(lock); + + return rc; +} + +/** Wait for the condition to become true with a locked irq spinlock. + * + * @param cv Condition variable. + * @param lock Locked irq spinlock. + * @param usec Timeout value in microseconds. + * @param flags Select mode of operation. + * + * For exact description of meaning of possible combinations of usec and flags, + * see comment for waitq_sleep_timeout(). Note that when + * SYNCH_FLAGS_NON_BLOCKING is specified here, ESYNCH_WOULD_BLOCK is always + * returned. + * + * @return See comment for waitq_sleep_timeout(). + */ +int _condvar_wait_timeout_irq_spinlock(condvar_t *cv, irq_spinlock_t *irq_lock, + uint32_t usec, int flags) +{ + int rc; + /* Save spinlock's state so we can restore it correctly later on. */ + ipl_t ipl = irq_lock->ipl; + bool guard = irq_lock->guard; + + irq_lock->guard = false; + + /* + * waitq_prepare() restores interrupts to the current state, + * ie disabled. Therefore, interrupts will remain disabled while + * it spins waiting for a pending timeout handler to complete. + * Although it spins with interrupts disabled there can only + * be a pending timeout if we failed to cancel an imminent + * timeout (on another cpu) during a wakeup. As a result the + * timeout handler is guaranteed to run (it is most likely already + * running) and there is no danger of a deadlock. + */ + rc = _condvar_wait_timeout_spinlock(cv, &irq_lock->lock, usec, flags); + + irq_lock->guard = guard; + irq_lock->ipl = ipl; + + return rc; +} + + /** @} */ Index: kernel/generic/src/synch/mutex.c =================================================================== --- kernel/generic/src/synch/mutex.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/synch/mutex.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,6 @@ #include #include +#include +#include /** Initialize mutex. Index: kernel/generic/src/synch/rcu.c =================================================================== --- kernel/generic/src/synch/rcu.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/src/synch/rcu.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,1749 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/** @addtogroup sync + * @{ + */ + +/** + * @file + * @brief Preemptible read-copy update. Usable from interrupt handlers. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Number of milliseconds to give to preexisting readers to finish + * when non-expedited grace period detection is in progress. + */ +#define DETECT_SLEEP_MS 10 +/* + * Max number of pending callbacks in the local cpu's queue before + * aggressively expediting the current grace period + */ +#define EXPEDITE_THRESHOLD 2000 +/* + * Max number of callbacks to execute in one go with preemption + * enabled. If there are more callbacks to be executed they will + * be run with preemption disabled in order to prolong reclaimer's + * time slice and give it a chance to catch up with callback producers. + */ +#define CRITICAL_THRESHOLD 30000 +/* Half the number of values a uint32 can hold. */ +#define UINT32_MAX_HALF 2147483648U + +/** + * The current grace period number. Increases monotonically. + * Lock rcu.gp_lock or rcu.preempt_lock to get a current value. + */ +rcu_gp_t _rcu_cur_gp; + +/** Global RCU data. */ +typedef struct rcu_data { + /** Detector uses so signal reclaimers that a grace period ended. */ + condvar_t gp_ended; + /** Reclaimers use to notify the detector to accelerate GP detection. */ + condvar_t expedite_now; + /** + * Protects: req_gp_end_cnt, req_expedited_cnt, completed_gp, _rcu_cur_gp; + * or: completed_gp, _rcu_cur_gp + */ + SPINLOCK_DECLARE(gp_lock); + /** + * The number of the most recently completed grace period. At most + * one behind _rcu_cur_gp. If equal to _rcu_cur_gp, a grace period + * detection is not in progress and the detector is idle. + */ + rcu_gp_t completed_gp; + + /** Protects the following 3 fields. */ + IRQ_SPINLOCK_DECLARE(preempt_lock); + /** Preexisting readers that have been preempted. */ + list_t cur_preempted; + /** Reader that have been preempted and might delay the next grace period.*/ + list_t next_preempted; + /** + * The detector is waiting for the last preempted reader + * in cur_preempted to announce that it exited its reader + * section by up()ing remaining_readers. + */ + bool preempt_blocking_det; + +#ifdef RCU_PREEMPT_A + + /** + * The detector waits on this semaphore for any preempted readers + * delaying the grace period once all cpus pass a quiescent state. + */ + semaphore_t remaining_readers; + +#elif defined(RCU_PREEMPT_PODZIMEK) + + /** Reclaimers notify the detector when they request more grace periods.*/ + condvar_t req_gp_changed; + /** Number of grace period ends the detector was requested to announce. */ + size_t req_gp_end_cnt; + /** Number of consecutive grace periods to detect quickly and aggressively.*/ + size_t req_expedited_cnt; + /** + * Number of cpus with readers that are delaying the current GP. + * They will up() remaining_readers. + */ + atomic_t delaying_cpu_cnt; + /** + * The detector waits on this semaphore for any readers delaying the GP. + * + * Each of the cpus with readers that are delaying the current GP + * must up() this sema once they reach a quiescent state. If there + * are any readers in cur_preempted (ie preempted preexisting) and + * they are already delaying GP detection, the last to unlock its + * reader section must up() this sema once. + */ + semaphore_t remaining_readers; +#endif + + /** Excludes simultaneous rcu_barrier() calls. */ + mutex_t barrier_mtx; + /** Number of cpus that we are waiting for to complete rcu_barrier(). */ + atomic_t barrier_wait_cnt; + /** rcu_barrier() waits for the completion of barrier callbacks on this wq.*/ + waitq_t barrier_wq; + + /** Interruptible attached detector thread pointer. */ + thread_t *detector_thr; + + /* Some statistics. */ + size_t stat_expedited_cnt; + size_t stat_delayed_cnt; + size_t stat_preempt_blocking_cnt; + /* Does not contain self/local calls. */ + size_t stat_smp_call_cnt; +} rcu_data_t; + + +static rcu_data_t rcu; + +static void start_reclaimers(void); +static void synch_complete(rcu_item_t *rcu_item); +static inline void rcu_call_impl(bool expedite, rcu_item_t *rcu_item, + rcu_func_t func); +static void add_barrier_cb(void *arg); +static void barrier_complete(rcu_item_t *barrier_item); +static bool arriving_cbs_empty(void); +static bool next_cbs_empty(void); +static bool cur_cbs_empty(void); +static bool all_cbs_empty(void); +static void reclaimer(void *arg); +static bool wait_for_pending_cbs(void); +static bool advance_cbs(void); +static void exec_completed_cbs(rcu_gp_t last_completed_gp); +static void exec_cbs(rcu_item_t **phead); +static bool wait_for_cur_cbs_gp_end(bool expedite, rcu_gp_t *last_completed_gp); +static void upd_missed_gp_in_wait(rcu_gp_t completed_gp); + +#ifdef RCU_PREEMPT_PODZIMEK +static void start_detector(void); +static void read_unlock_impl(size_t *pnesting_cnt); +static void req_detection(size_t req_cnt); +static bool cv_wait_for_gp(rcu_gp_t wait_on_gp); +static void detector(void *); +static bool wait_for_detect_req(void); +static void end_cur_gp(void); +static bool wait_for_readers(void); +static bool gp_sleep(void); +static void interrupt_delaying_cpus(cpu_mask_t *cpu_mask); +static bool wait_for_delaying_cpus(void); +#elif defined(RCU_PREEMPT_A) +static bool wait_for_readers(bool expedite); +static bool gp_sleep(bool *expedite); +#endif + +static void start_new_gp(void); +static void rm_quiescent_cpus(cpu_mask_t *cpu_mask); +static void sample_cpus(cpu_mask_t *reader_cpus, void *arg); +static void sample_local_cpu(void *); +static bool wait_for_preempt_reader(void); +static void note_preempted_reader(void); +static void rm_preempted_reader(void); +static void upd_max_cbs_in_slice(void); + + + +/** Initializes global RCU structures. */ +void rcu_init(void) +{ + condvar_initialize(&rcu.gp_ended); + condvar_initialize(&rcu.expedite_now); + + spinlock_initialize(&rcu.gp_lock, "rcu.gp_lock"); + _rcu_cur_gp = 0; + rcu.completed_gp = 0; + + irq_spinlock_initialize(&rcu.preempt_lock, "rcu.preempt_lock"); + list_initialize(&rcu.cur_preempted); + list_initialize(&rcu.next_preempted); + rcu.preempt_blocking_det = false; + + mutex_initialize(&rcu.barrier_mtx, MUTEX_PASSIVE); + atomic_set(&rcu.barrier_wait_cnt, 0); + waitq_initialize(&rcu.barrier_wq); + + semaphore_initialize(&rcu.remaining_readers, 0); + +#ifdef RCU_PREEMPT_PODZIMEK + condvar_initialize(&rcu.req_gp_changed); + + rcu.req_gp_end_cnt = 0; + rcu.req_expedited_cnt = 0; + atomic_set(&rcu.delaying_cpu_cnt, 0); +#endif + + rcu.detector_thr = 0; + + rcu.stat_expedited_cnt = 0; + rcu.stat_delayed_cnt = 0; + rcu.stat_preempt_blocking_cnt = 0; + rcu.stat_smp_call_cnt = 0; +} + +/** Initializes per-CPU RCU data. If on the boot cpu inits global data too.*/ +void rcu_cpu_init(void) +{ + if (config.cpu_active == 1) { + rcu_init(); + } + + CPU->rcu.last_seen_gp = 0; + +#ifdef RCU_PREEMPT_PODZIMEK + CPU->rcu.nesting_cnt = 0; + CPU->rcu.is_delaying_gp = false; + CPU->rcu.signal_unlock = false; +#endif + + CPU->rcu.cur_cbs = 0; + CPU->rcu.cur_cbs_cnt = 0; + CPU->rcu.next_cbs = 0; + CPU->rcu.next_cbs_cnt = 0; + CPU->rcu.arriving_cbs = 0; + CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs; + CPU->rcu.arriving_cbs_cnt = 0; + + CPU->rcu.cur_cbs_gp = 0; + CPU->rcu.next_cbs_gp = 0; + + semaphore_initialize(&CPU->rcu.arrived_flag, 0); + + /* BSP creates reclaimer threads before AP's rcu_cpu_init() runs. */ + if (config.cpu_active == 1) + CPU->rcu.reclaimer_thr = 0; + + CPU->rcu.stat_max_cbs = 0; + CPU->rcu.stat_avg_cbs = 0; + CPU->rcu.stat_missed_gps = 0; + CPU->rcu.stat_missed_gp_in_wait = 0; + CPU->rcu.stat_max_slice_cbs = 0; + CPU->rcu.last_arriving_cnt = 0; +} + +/** Completes RCU init. Creates and runs the detector and reclaimer threads.*/ +void rcu_kinit_init(void) +{ +#ifdef RCU_PREEMPT_PODZIMEK + start_detector(); +#endif + + start_reclaimers(); +} + +/** Initializes any per-thread RCU structures. */ +void rcu_thread_init(thread_t *thread) +{ + thread->rcu.nesting_cnt = 0; + +#ifdef RCU_PREEMPT_PODZIMEK + thread->rcu.was_preempted = false; +#endif + + link_initialize(&thread->rcu.preempt_link); +} + + +/** Cleans up global RCU resources and stops dispatching callbacks. + * + * Call when shutting down the kernel. Outstanding callbacks will + * not be processed. Instead they will linger forever. + */ +void rcu_stop(void) +{ + /* Stop and wait for reclaimers. */ + for (unsigned int cpu_id = 0; cpu_id < config.cpu_active; ++cpu_id) { + ASSERT(cpus[cpu_id].rcu.reclaimer_thr != 0); + + if (cpus[cpu_id].rcu.reclaimer_thr) { + thread_interrupt(cpus[cpu_id].rcu.reclaimer_thr); + thread_join(cpus[cpu_id].rcu.reclaimer_thr); + thread_detach(cpus[cpu_id].rcu.reclaimer_thr); + cpus[cpu_id].rcu.reclaimer_thr = 0; + } + } + +#ifdef RCU_PREEMPT_PODZIMEK + /* Stop the detector and wait. */ + if (rcu.detector_thr) { + thread_interrupt(rcu.detector_thr); + thread_join(rcu.detector_thr); + thread_detach(rcu.detector_thr); + rcu.detector_thr = 0; + } +#endif +} + +/** Returns the number of elapsed grace periods since boot. */ +uint64_t rcu_completed_gps(void) +{ + spinlock_lock(&rcu.gp_lock); + uint64_t completed = rcu.completed_gp; + spinlock_unlock(&rcu.gp_lock); + + return completed; +} + +/** Creates and runs cpu-bound reclaimer threads. */ +static void start_reclaimers(void) +{ + for (unsigned int cpu_id = 0; cpu_id < config.cpu_count; ++cpu_id) { + char name[THREAD_NAME_BUFLEN] = {0}; + + snprintf(name, THREAD_NAME_BUFLEN - 1, "rcu-rec/%u", cpu_id); + + cpus[cpu_id].rcu.reclaimer_thr = + thread_create(reclaimer, 0, TASK, THREAD_FLAG_NONE, name); + + if (!cpus[cpu_id].rcu.reclaimer_thr) + panic("Failed to create RCU reclaimer thread on cpu%u.", cpu_id); + + thread_wire(cpus[cpu_id].rcu.reclaimer_thr, &cpus[cpu_id]); + thread_ready(cpus[cpu_id].rcu.reclaimer_thr); + } +} + +#ifdef RCU_PREEMPT_PODZIMEK + +/** Starts the detector thread. */ +static void start_detector(void) +{ + rcu.detector_thr = + thread_create(detector, 0, TASK, THREAD_FLAG_NONE, "rcu-det"); + + if (!rcu.detector_thr) + panic("Failed to create RCU detector thread."); + + thread_ready(rcu.detector_thr); +} + +/** Returns true if in an rcu reader section. */ +bool rcu_read_locked(void) +{ + preemption_disable(); + bool locked = 0 < CPU->rcu.nesting_cnt; + preemption_enable(); + + return locked; +} + +/** Unlocks the local reader section using the given nesting count. + * + * Preemption or interrupts must be disabled. + * + * @param pnesting_cnt Either &CPU->rcu.tmp_nesting_cnt or + * THREAD->rcu.nesting_cnt. + */ +static void read_unlock_impl(size_t *pnesting_cnt) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + + if (0 == --(*pnesting_cnt)) { + _rcu_record_qs(); + + /* + * The thread was preempted while in a critical section or + * the detector is eagerly waiting for this cpu's reader + * to finish. + * + * Note that THREAD may be 0 in scheduler() and not just during boot. + */ + if ((THREAD && THREAD->rcu.was_preempted) || CPU->rcu.is_delaying_gp) { + /* Rechecks with disabled interrupts. */ + _rcu_signal_read_unlock(); + } + } +} + +/** If necessary, signals the detector that we exited a reader section. */ +void _rcu_signal_read_unlock(void) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + + /* todo: make NMI safe with cpu-local atomic ops. */ + + /* + * We have to disable interrupts in order to make checking + * and resetting was_preempted and is_delaying_gp atomic + * with respect to local interrupt handlers. Otherwise + * an interrupt could beat us to calling semaphore_up() + * before we reset the appropriate flag. + */ + ipl_t ipl = interrupts_disable(); + + /* + * If the detector is eagerly waiting for this cpu's reader to unlock, + * notify it that the reader did so. + */ + if (CPU->rcu.is_delaying_gp) { + CPU->rcu.is_delaying_gp = false; + semaphore_up(&rcu.remaining_readers); + } + + /* + * This reader was preempted while in a reader section. + * We might be holding up the current GP. Notify the + * detector if so. + */ + if (THREAD && THREAD->rcu.was_preempted) { + ASSERT(link_used(&THREAD->rcu.preempt_link)); + THREAD->rcu.was_preempted = false; + + rm_preempted_reader(); + } + + /* If there was something to signal to the detector we have done so. */ + CPU->rcu.signal_unlock = false; + + interrupts_restore(ipl); +} + +#endif /* RCU_PREEMPT_PODZIMEK */ + +typedef struct synch_item { + waitq_t wq; + rcu_item_t rcu_item; +} synch_item_t; + +/** Blocks until all preexisting readers exit their critical sections. */ +void rcu_synchronize(void) +{ + _rcu_synchronize(false); +} + +/** Blocks until all preexisting readers exit their critical sections. */ +void rcu_synchronize_expedite(void) +{ + _rcu_synchronize(true); +} + +/** Blocks until all preexisting readers exit their critical sections. */ +void _rcu_synchronize(bool expedite) +{ + /* Calling from a reader section will deadlock. */ + ASSERT(!rcu_read_locked()); + + synch_item_t completion; + + waitq_initialize(&completion.wq); + _rcu_call(expedite, &completion.rcu_item, synch_complete); + waitq_sleep(&completion.wq); + waitq_complete_wakeup(&completion.wq); +} + +/** rcu_synchronize's callback. */ +static void synch_complete(rcu_item_t *rcu_item) +{ + synch_item_t *completion = member_to_inst(rcu_item, synch_item_t, rcu_item); + ASSERT(completion); + waitq_wakeup(&completion->wq, WAKEUP_FIRST); +} + +/** Waits for all outstanding rcu calls to complete. */ +void rcu_barrier(void) +{ + /* + * Serialize rcu_barrier() calls so we don't overwrite cpu.barrier_item + * currently in use by rcu_barrier(). + */ + mutex_lock(&rcu.barrier_mtx); + + /* + * Ensure we queue a barrier callback on all cpus before the already + * enqueued barrier callbacks start signaling completion. + */ + atomic_set(&rcu.barrier_wait_cnt, 1); + + DEFINE_CPU_MASK(cpu_mask); + cpu_mask_active(cpu_mask); + + cpu_mask_for_each(*cpu_mask, cpu_id) { + smp_call(cpu_id, add_barrier_cb, 0); + } + + if (0 < atomic_predec(&rcu.barrier_wait_cnt)) { + waitq_sleep(&rcu.barrier_wq); + } + + mutex_unlock(&rcu.barrier_mtx); +} + +/** Issues a rcu_barrier() callback on the local cpu. + * + * Executed with interrupts disabled. + */ +static void add_barrier_cb(void *arg) +{ + ASSERT(interrupts_disabled() || PREEMPTION_DISABLED); + atomic_inc(&rcu.barrier_wait_cnt); + rcu_call(&CPU->rcu.barrier_item, barrier_complete); +} + +/** Local cpu's rcu_barrier() completion callback. */ +static void barrier_complete(rcu_item_t *barrier_item) +{ + /* Is this the last barrier callback completed? */ + if (0 == atomic_predec(&rcu.barrier_wait_cnt)) { + /* Notify rcu_barrier() that we're done. */ + waitq_wakeup(&rcu.barrier_wq, WAKEUP_FIRST); + } +} + +/** Adds a callback to invoke after all preexisting readers finish. + * + * May be called from within interrupt handlers or RCU reader sections. + * + * @param rcu_item Used by RCU to track the call. Must remain + * until the user callback function is entered. + * @param func User callback function that will be invoked once a full + * grace period elapsed, ie at a time when all preexisting + * readers have finished. The callback should be short and must + * not block. If you must sleep, enqueue your work in the system + * work queue from the callback (ie workq_global_enqueue()). + */ +void rcu_call(rcu_item_t *rcu_item, rcu_func_t func) +{ + rcu_call_impl(false, rcu_item, func); +} + +/** rcu_call() implementation. See rcu_call() for comments. */ +void _rcu_call(bool expedite, rcu_item_t *rcu_item, rcu_func_t func) +{ + rcu_call_impl(expedite, rcu_item, func); +} + +/** rcu_call() inline-able implementation. See rcu_call() for comments. */ +static inline void rcu_call_impl(bool expedite, rcu_item_t *rcu_item, + rcu_func_t func) +{ + ASSERT(rcu_item); + + rcu_item->func = func; + rcu_item->next = 0; + + preemption_disable(); + + ipl_t ipl = interrupts_disable(); + + rcu_cpu_data_t *r = &CPU->rcu; + *r->parriving_cbs_tail = rcu_item; + r->parriving_cbs_tail = &rcu_item->next; + + size_t cnt = ++r->arriving_cbs_cnt; + interrupts_restore(ipl); + + if (expedite) { + r->expedite_arriving = true; + } + + /* Added first callback - notify the reclaimer. */ + if (cnt == 1 && !semaphore_count_get(&r->arrived_flag)) { + semaphore_up(&r->arrived_flag); + } + + preemption_enable(); +} + +static bool cur_cbs_empty(void) +{ + ASSERT(THREAD && THREAD->wired); + return 0 == CPU->rcu.cur_cbs; +} + +static bool next_cbs_empty(void) +{ + ASSERT(THREAD && THREAD->wired); + return 0 == CPU->rcu.next_cbs; +} + +/** Disable interrupts to get an up-to-date result. */ +static bool arriving_cbs_empty(void) +{ + ASSERT(THREAD && THREAD->wired); + /* + * Accessing with interrupts enabled may at worst lead to + * a false negative if we race with a local interrupt handler. + */ + return 0 == CPU->rcu.arriving_cbs; +} + +static bool all_cbs_empty(void) +{ + return cur_cbs_empty() && next_cbs_empty() && arriving_cbs_empty(); +} + + +/** Reclaimer thread dispatches locally queued callbacks once a GP ends. */ +static void reclaimer(void *arg) +{ + ASSERT(THREAD && THREAD->wired); + ASSERT(THREAD == CPU->rcu.reclaimer_thr); + + rcu_gp_t last_compl_gp = 0; + bool ok = true; + + while (ok && wait_for_pending_cbs()) { + ASSERT(CPU->rcu.reclaimer_thr == THREAD); + + exec_completed_cbs(last_compl_gp); + + bool expedite = advance_cbs(); + + ok = wait_for_cur_cbs_gp_end(expedite, &last_compl_gp); + } +} + +/** Waits until there are callbacks waiting to be dispatched. */ +static bool wait_for_pending_cbs(void) +{ + if (!all_cbs_empty()) + return true; + + bool ok = true; + + while (arriving_cbs_empty() && ok) { + ok = semaphore_down_interruptable(&CPU->rcu.arrived_flag); + } + + return ok; +} + +static void upd_stat_missed_gp(rcu_gp_t compl) +{ + if (CPU->rcu.cur_cbs_gp < compl) { + CPU->rcu.stat_missed_gps += (size_t)(compl - CPU->rcu.cur_cbs_gp); + } +} + +/** Executes all callbacks for the given completed grace period. */ +static void exec_completed_cbs(rcu_gp_t last_completed_gp) +{ + upd_stat_missed_gp(last_completed_gp); + + /* Both next_cbs and cur_cbs GP elapsed. */ + if (CPU->rcu.next_cbs_gp <= last_completed_gp) { + ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp); + + size_t exec_cnt = CPU->rcu.cur_cbs_cnt + CPU->rcu.next_cbs_cnt; + + if (exec_cnt < CRITICAL_THRESHOLD) { + exec_cbs(&CPU->rcu.cur_cbs); + exec_cbs(&CPU->rcu.next_cbs); + } else { + /* + * Getting overwhelmed with too many callbacks to run. + * Disable preemption in order to prolong our time slice + * and catch up with updaters posting new callbacks. + */ + preemption_disable(); + exec_cbs(&CPU->rcu.cur_cbs); + exec_cbs(&CPU->rcu.next_cbs); + preemption_enable(); + } + + CPU->rcu.cur_cbs_cnt = 0; + CPU->rcu.next_cbs_cnt = 0; + } else if (CPU->rcu.cur_cbs_gp <= last_completed_gp) { + + if (CPU->rcu.cur_cbs_cnt < CRITICAL_THRESHOLD) { + exec_cbs(&CPU->rcu.cur_cbs); + } else { + /* + * Getting overwhelmed with too many callbacks to run. + * Disable preemption in order to prolong our time slice + * and catch up with updaters posting new callbacks. + */ + preemption_disable(); + exec_cbs(&CPU->rcu.cur_cbs); + preemption_enable(); + } + + CPU->rcu.cur_cbs_cnt = 0; + } +} + +/** Executes callbacks in the single-linked list. The list is left empty. */ +static void exec_cbs(rcu_item_t **phead) +{ + rcu_item_t *rcu_item = *phead; + + while (rcu_item) { + /* func() may free rcu_item. Get a local copy. */ + rcu_item_t *next = rcu_item->next; + rcu_func_t func = rcu_item->func; + + func(rcu_item); + + rcu_item = next; + } + + *phead = 0; +} + +static void upd_stat_cb_cnts(size_t arriving_cnt) +{ + CPU->rcu.stat_max_cbs = max(arriving_cnt, CPU->rcu.stat_max_cbs); + if (0 < arriving_cnt) { + CPU->rcu.stat_avg_cbs = + (99 * CPU->rcu.stat_avg_cbs + 1 * arriving_cnt) / 100; + } +} + +/** Prepares another batch of callbacks to dispatch at the nest grace period. + * + * @return True if the next batch of callbacks must be expedited quickly. + */ +static bool advance_cbs(void) +{ + /* Move next_cbs to cur_cbs. */ + CPU->rcu.cur_cbs = CPU->rcu.next_cbs; + CPU->rcu.cur_cbs_cnt = CPU->rcu.next_cbs_cnt; + CPU->rcu.cur_cbs_gp = CPU->rcu.next_cbs_gp; + + /* Move arriving_cbs to next_cbs. Empties arriving_cbs. */ + ipl_t ipl = interrupts_disable(); + + /* + * Too many callbacks queued. Better speed up the detection + * or risk exhausting all system memory. + */ + bool expedite = (EXPEDITE_THRESHOLD < CPU->rcu.arriving_cbs_cnt) + || CPU->rcu.expedite_arriving; + + CPU->rcu.expedite_arriving = false; + + CPU->rcu.next_cbs = CPU->rcu.arriving_cbs; + CPU->rcu.next_cbs_cnt = CPU->rcu.arriving_cbs_cnt; + + CPU->rcu.arriving_cbs = 0; + CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs; + CPU->rcu.arriving_cbs_cnt = 0; + + interrupts_restore(ipl); + + /* Update statistics of arrived callbacks. */ + upd_stat_cb_cnts(CPU->rcu.next_cbs_cnt); + + /* + * Make changes prior to queuing next_cbs visible to readers. + * See comment in wait_for_readers(). + */ + memory_barrier(); /* MB A, B */ + + /* At the end of next_cbs_gp, exec next_cbs. Determine what GP that is. */ + + if (!next_cbs_empty()) { + spinlock_lock(&rcu.gp_lock); + + /* Exec next_cbs at the end of the next GP. */ + CPU->rcu.next_cbs_gp = _rcu_cur_gp + 1; + + /* + * There are no callbacks to invoke before next_cbs. Instruct + * wait_for_cur_cbs_gp() to notify us of the nearest GP end. + * That could be sooner than next_cbs_gp (if the current GP + * had not yet completed), so we'll create a shorter batch + * of callbacks next time around. + */ + if (cur_cbs_empty()) { + CPU->rcu.cur_cbs_gp = rcu.completed_gp + 1; + } + + spinlock_unlock(&rcu.gp_lock); + } else { + CPU->rcu.next_cbs_gp = CPU->rcu.cur_cbs_gp; + } + + ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp); + + return expedite; +} + + +#ifdef RCU_PREEMPT_A + +/** Waits for the grace period associated with callbacks cub_cbs to elapse. + * + * @param expedite Instructs the detector to aggressively speed up grace + * period detection without any delay. + * @param completed_gp Returns the most recent completed grace period + * number. + * @return false if the thread was interrupted and should stop. + */ +static bool wait_for_cur_cbs_gp_end(bool expedite, rcu_gp_t *completed_gp) +{ + spinlock_lock(&rcu.gp_lock); + + ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp); + ASSERT(CPU->rcu.cur_cbs_gp <= _rcu_cur_gp + 1); + + while (rcu.completed_gp < CPU->rcu.cur_cbs_gp) { + /* GP has not yet started - start a new one. */ + if (rcu.completed_gp == _rcu_cur_gp) { + start_new_gp(); + spinlock_unlock(&rcu.gp_lock); + + if (!wait_for_readers(expedite)) + return false; + + spinlock_lock(&rcu.gp_lock); + /* Notify any reclaimers this GP had ended. */ + rcu.completed_gp = _rcu_cur_gp; + condvar_broadcast(&rcu.gp_ended); + } else { + /* GP detection is in progress.*/ + + if (expedite) + condvar_signal(&rcu.expedite_now); + + /* Wait for the GP to complete. */ + int ret = _condvar_wait_timeout_spinlock(&rcu.gp_ended, + &rcu.gp_lock, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_INTERRUPTIBLE); + + if (ret == ESYNCH_INTERRUPTED) { + spinlock_unlock(&rcu.gp_lock); + return false; + } + } + } + + upd_missed_gp_in_wait(rcu.completed_gp); + + *completed_gp = rcu.completed_gp; + spinlock_unlock(&rcu.gp_lock); + + return true; +} + +static bool wait_for_readers(bool expedite) +{ + DEFINE_CPU_MASK(reader_cpus); + + cpu_mask_active(reader_cpus); + rm_quiescent_cpus(reader_cpus); + + while (!cpu_mask_is_none(reader_cpus)) { + /* Give cpus a chance to context switch (a QS) and batch callbacks. */ + if(!gp_sleep(&expedite)) + return false; + + rm_quiescent_cpus(reader_cpus); + sample_cpus(reader_cpus, reader_cpus); + } + + /* Update statistic. */ + if (expedite) { + ++rcu.stat_expedited_cnt; + } + + /* + * All cpus have passed through a QS and see the most recent _rcu_cur_gp. + * As a result newly preempted readers will associate with next_preempted + * and the number of old readers in cur_preempted will monotonically + * decrease. Wait for those old/preexisting readers. + */ + return wait_for_preempt_reader(); +} + +static bool gp_sleep(bool *expedite) +{ + if (*expedite) { + scheduler(); + return true; + } else { + spinlock_lock(&rcu.gp_lock); + + int ret = 0; + ret = _condvar_wait_timeout_spinlock(&rcu.expedite_now, &rcu.gp_lock, + DETECT_SLEEP_MS * 1000, SYNCH_FLAGS_INTERRUPTIBLE); + + /* rcu.expedite_now was signaled. */ + if (ret == ESYNCH_OK_BLOCKED) { + *expedite = true; + } + + spinlock_unlock(&rcu.gp_lock); + + return (ret != ESYNCH_INTERRUPTED); + } +} + +static void sample_local_cpu(void *arg) +{ + ASSERT(interrupts_disabled()); + cpu_mask_t *reader_cpus = (cpu_mask_t *)arg; + + bool locked = RCU_CNT_INC <= THE->rcu_nesting; + bool passed_qs = (CPU->rcu.last_seen_gp == _rcu_cur_gp); + + if (locked && !passed_qs) { + /* + * This cpu has not yet passed a quiescent state during this grace + * period and it is currently in a reader section. We'll have to + * try to sample this cpu again later. + */ + } else { + /* Either not in a reader section or already passed a QS. */ + cpu_mask_reset(reader_cpus, CPU->id); + /* Contain new reader sections and make prior changes visible to them.*/ + memory_barrier(); + CPU->rcu.last_seen_gp = _rcu_cur_gp; + } +} + +/** Called by the scheduler() when switching away from the current thread. */ +void rcu_after_thread_ran(void) +{ + ASSERT(interrupts_disabled()); + + /* Preempted a reader critical section for the first time. */ + if (rcu_read_locked() && !(THE->rcu_nesting & RCU_WAS_PREEMPTED)) { + THE->rcu_nesting |= RCU_WAS_PREEMPTED; + note_preempted_reader(); + } + + /* Save the thread's nesting count when it is not running. */ + THREAD->rcu.nesting_cnt = THE->rcu_nesting; + + /* Clear rcu_nesting only after noting that a thread was preempted. */ + compiler_barrier(); + THE->rcu_nesting = 0; + + if (CPU->rcu.last_seen_gp != _rcu_cur_gp) { + /* + * Contain any memory accesses of old readers before announcing a QS. + * Also make changes from the previous GP visible to this cpu. + */ + memory_barrier(); + /* + * The preempted reader has been noted globally. There are therefore + * no readers running on this cpu so this is a quiescent state. + */ + CPU->rcu.last_seen_gp = _rcu_cur_gp; + } + + /* + * Forcefully associate the reclaime with the highest priority + * even if preempted due to its time slice running out. + */ + if (THREAD == CPU->rcu.reclaimer_thr) { + THREAD->priority = -1; + } + + upd_max_cbs_in_slice(); +} + +/** Called by the scheduler() when switching to a newly scheduled thread. */ +void rcu_before_thread_runs(void) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + ASSERT(!rcu_read_locked()); + + /* Load the thread's saved nesting count from before it was preempted. */ + THE->rcu_nesting = THREAD->rcu.nesting_cnt; +} + +/** Called from scheduler() when exiting the current thread. + * + * Preemption or interrupts are disabled and the scheduler() already + * switched away from the current thread, calling rcu_after_thread_ran(). + */ +void rcu_thread_exiting(void) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + /* + * The thread forgot to exit its reader critical section. + * It is a bug, but rather than letting the entire system lock up + * forcefully leave the reader section. The thread is not holding + * any references anyway since it is exiting so it is safe. + */ + if (RCU_CNT_INC <= THREAD->rcu.nesting_cnt) { + /* Emulate _rcu_preempted_unlock() with the proper nesting count. */ + if (THREAD->rcu.nesting_cnt & RCU_WAS_PREEMPTED) { + ipl_t ipl = interrupts_disable(); + rm_preempted_reader(); + interrupts_restore(ipl); + } + + printf("Bug: thread (id %" PRIu64 " \"%s\") exited while in RCU read" + " section.\n", THREAD->tid, THREAD->name); + } +} + +/** Returns true if in an rcu reader section. */ +bool rcu_read_locked(void) +{ + return RCU_CNT_INC <= THE->rcu_nesting; +} + +/** Invoked when a preempted reader finally exits its reader section. */ +void _rcu_preempted_unlock(void) +{ + ipl_t ipl = interrupts_disable(); + + if (THE->rcu_nesting == RCU_WAS_PREEMPTED) { + THE->rcu_nesting = 0; + rm_preempted_reader(); + } + + interrupts_restore(ipl); +} + +#elif defined(RCU_PREEMPT_PODZIMEK) + +/** Waits for the grace period associated with callbacks cub_cbs to elapse. + * + * @param expedite Instructs the detector to aggressively speed up grace + * period detection without any delay. + * @param completed_gp Returns the most recent completed grace period + * number. + * @return false if the thread was interrupted and should stop. + */ +static bool wait_for_cur_cbs_gp_end(bool expedite, rcu_gp_t *completed_gp) +{ + /* + * Use a possibly outdated version of completed_gp to bypass checking + * with the lock. + * + * Note that loading and storing rcu.completed_gp is not atomic + * (it is 64bit wide). Reading a clobbered value that is less than + * rcu.completed_gp is harmless - we'll recheck with a lock. The + * only way to read a clobbered value that is greater than the actual + * value is if the detector increases the higher-order word first and + * then decreases the lower-order word (or we see stores in that order), + * eg when incrementing from 2^32 - 1 to 2^32. The loaded value + * suddenly jumps by 2^32. It would take hours for such an increase + * to occur so it is safe to discard the value. We allow increases + * of up to half the maximum to generously accommodate for loading an + * outdated lower word. + */ + rcu_gp_t compl_gp = ACCESS_ONCE(rcu.completed_gp); + if (CPU->rcu.cur_cbs_gp <= compl_gp + && compl_gp <= CPU->rcu.cur_cbs_gp + UINT32_MAX_HALF) { + *completed_gp = compl_gp; + return true; + } + + spinlock_lock(&rcu.gp_lock); + + if (CPU->rcu.cur_cbs_gp <= rcu.completed_gp) { + *completed_gp = rcu.completed_gp; + spinlock_unlock(&rcu.gp_lock); + return true; + } + + ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp); + ASSERT(_rcu_cur_gp <= CPU->rcu.cur_cbs_gp); + + /* + * Notify the detector of how many GP ends we intend to wait for, so + * it can avoid going to sleep unnecessarily. Optimistically assume + * new callbacks will arrive while we're waiting; hence +1. + */ + size_t remaining_gp_ends = (size_t) (CPU->rcu.next_cbs_gp - _rcu_cur_gp); + req_detection(remaining_gp_ends + (arriving_cbs_empty() ? 0 : 1)); + + /* + * Ask the detector to speed up GP detection if there are too many + * pending callbacks and other reclaimers have not already done so. + */ + if (expedite) { + if(0 == rcu.req_expedited_cnt) + condvar_signal(&rcu.expedite_now); + + /* + * Expedite only cub_cbs. If there really is a surge of callbacks + * the arriving batch will expedite the GP for the huge number + * of callbacks currently in next_cbs + */ + rcu.req_expedited_cnt = 1; + } + + /* Wait for cur_cbs_gp to end. */ + bool interrupted = cv_wait_for_gp(CPU->rcu.cur_cbs_gp); + + *completed_gp = rcu.completed_gp; + spinlock_unlock(&rcu.gp_lock); + + if (!interrupted) + upd_missed_gp_in_wait(*completed_gp); + + return !interrupted; +} + +/** Waits for an announcement of the end of the grace period wait_on_gp. */ +static bool cv_wait_for_gp(rcu_gp_t wait_on_gp) +{ + ASSERT(spinlock_locked(&rcu.gp_lock)); + + bool interrupted = false; + + /* Wait until wait_on_gp ends. */ + while (rcu.completed_gp < wait_on_gp && !interrupted) { + int ret = _condvar_wait_timeout_spinlock(&rcu.gp_ended, &rcu.gp_lock, + SYNCH_NO_TIMEOUT, SYNCH_FLAGS_INTERRUPTIBLE); + interrupted = (ret == ESYNCH_INTERRUPTED); + } + + return interrupted; +} + +/** Requests the detector to detect at least req_cnt consecutive grace periods.*/ +static void req_detection(size_t req_cnt) +{ + if (rcu.req_gp_end_cnt < req_cnt) { + bool detector_idle = (0 == rcu.req_gp_end_cnt); + rcu.req_gp_end_cnt = req_cnt; + + if (detector_idle) { + ASSERT(_rcu_cur_gp == rcu.completed_gp); + condvar_signal(&rcu.req_gp_changed); + } + } +} + + +/** The detector thread detects and notifies reclaimers of grace period ends. */ +static void detector(void *arg) +{ + spinlock_lock(&rcu.gp_lock); + + while (wait_for_detect_req()) { + /* + * Announce new GP started. Readers start lazily acknowledging that + * they passed a QS. + */ + start_new_gp(); + + spinlock_unlock(&rcu.gp_lock); + + if (!wait_for_readers()) + goto unlocked_out; + + spinlock_lock(&rcu.gp_lock); + + /* Notify reclaimers that they may now invoke queued callbacks. */ + end_cur_gp(); + } + + spinlock_unlock(&rcu.gp_lock); + +unlocked_out: + return; +} + +/** Waits for a request from a reclaimer thread to detect a grace period. */ +static bool wait_for_detect_req(void) +{ + ASSERT(spinlock_locked(&rcu.gp_lock)); + + bool interrupted = false; + + while (0 == rcu.req_gp_end_cnt && !interrupted) { + int ret = _condvar_wait_timeout_spinlock(&rcu.req_gp_changed, + &rcu.gp_lock, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_INTERRUPTIBLE); + + interrupted = (ret == ESYNCH_INTERRUPTED); + } + + return !interrupted; +} + + +static void end_cur_gp(void) +{ + ASSERT(spinlock_locked(&rcu.gp_lock)); + + rcu.completed_gp = _rcu_cur_gp; + --rcu.req_gp_end_cnt; + + condvar_broadcast(&rcu.gp_ended); +} + +/** Waits for readers that started before the current GP started to finish. */ +static bool wait_for_readers(void) +{ + DEFINE_CPU_MASK(reading_cpus); + + /* All running cpus have potential readers. */ + cpu_mask_active(reading_cpus); + + /* + * Give readers time to pass through a QS. Also, batch arriving + * callbacks in order to amortize detection overhead. + */ + if (!gp_sleep()) + return false; + + /* Non-intrusively determine which cpus have yet to pass a QS. */ + rm_quiescent_cpus(reading_cpus); + + /* Actively interrupt cpus delaying the current GP and demand a QS. */ + interrupt_delaying_cpus(reading_cpus); + + /* Wait for the interrupted cpus to notify us that they reached a QS. */ + if (!wait_for_delaying_cpus()) + return false; + /* + * All cpus recorded a QS or are still idle. Any new readers will be added + * to next_preempt if preempted, ie the number of readers in cur_preempted + * monotonically descreases. + */ + + /* Wait for the last reader in cur_preempted to notify us it is done. */ + if (!wait_for_preempt_reader()) + return false; + + return true; +} + +/** Sleeps a while if the current grace period is not to be expedited. */ +static bool gp_sleep(void) +{ + spinlock_lock(&rcu.gp_lock); + + int ret = 0; + while (0 == rcu.req_expedited_cnt && 0 == ret) { + /* minor bug: sleeps for the same duration if woken up spuriously. */ + ret = _condvar_wait_timeout_spinlock(&rcu.expedite_now, &rcu.gp_lock, + DETECT_SLEEP_MS * 1000, SYNCH_FLAGS_INTERRUPTIBLE); + } + + if (0 < rcu.req_expedited_cnt) { + --rcu.req_expedited_cnt; + /* Update statistic. */ + ++rcu.stat_expedited_cnt; + } + + spinlock_unlock(&rcu.gp_lock); + + return (ret != ESYNCH_INTERRUPTED); +} + +/** Actively interrupts and checks the offending cpus for quiescent states. */ +static void interrupt_delaying_cpus(cpu_mask_t *cpu_mask) +{ + atomic_set(&rcu.delaying_cpu_cnt, 0); + + sample_cpus(cpu_mask, 0); +} + +/** Invoked on a cpu delaying grace period detection. + * + * Induces a quiescent state for the cpu or it instructs remaining + * readers to notify the detector once they finish. + */ +static void sample_local_cpu(void *arg) +{ + ASSERT(interrupts_disabled()); + ASSERT(!CPU->rcu.is_delaying_gp); + + /* Cpu did not pass a quiescent state yet. */ + if (CPU->rcu.last_seen_gp != _rcu_cur_gp) { + /* Interrupted a reader in a reader critical section. */ + if (0 < CPU->rcu.nesting_cnt) { + ASSERT(!CPU->idle); + /* Note to notify the detector from rcu_read_unlock(). */ + CPU->rcu.is_delaying_gp = true; + /* + * Set signal_unlock only after setting is_delaying_gp so + * that NMI handlers do not accidentally clear it in unlock() + * before seeing and acting upon is_delaying_gp. + */ + compiler_barrier(); + CPU->rcu.signal_unlock = true; + + atomic_inc(&rcu.delaying_cpu_cnt); + } else { + /* + * The cpu did not enter any rcu reader sections since + * the start of the current GP. Record a quiescent state. + * + * Or, we interrupted rcu_read_unlock_impl() right before + * it recorded a QS. Record a QS for it. The memory barrier + * contains the reader section's mem accesses before + * updating last_seen_gp. + * + * Or, we interrupted rcu_read_lock() right after it recorded + * a QS for the previous GP but before it got a chance to + * increment its nesting count. The memory barrier again + * stops the CS code from spilling out of the CS. + */ + memory_barrier(); + CPU->rcu.last_seen_gp = _rcu_cur_gp; + } + } else { + /* + * This cpu already acknowledged that it had passed through + * a quiescent state since the start of cur_gp. + */ + } + + /* + * smp_call() makes sure any changes propagate back to the caller. + * In particular, it makes the most current last_seen_gp visible + * to the detector. + */ +} + +/** Waits for cpus delaying the current grace period if there are any. */ +static bool wait_for_delaying_cpus(void) +{ + int delaying_cpu_cnt = atomic_get(&rcu.delaying_cpu_cnt); + + for (int i = 0; i < delaying_cpu_cnt; ++i){ + if (!semaphore_down_interruptable(&rcu.remaining_readers)) + return false; + } + + /* Update statistic. */ + rcu.stat_delayed_cnt += delaying_cpu_cnt; + + return true; +} + +/** Called by the scheduler() when switching away from the current thread. */ +void rcu_after_thread_ran(void) +{ + ASSERT(interrupts_disabled()); + /* todo: make is_delaying_gp and was_preempted NMI safe via local atomics.*/ + + /* + * Prevent NMI handlers from interfering. The detector will be notified + * here if CPU->rcu.is_delaying_gp and the current thread is no longer + * running so there is nothing to signal to the detector. + */ + CPU->rcu.signal_unlock = false; + /* Separates clearing of .signal_unlock from CPU->rcu.nesting_cnt = 0. */ + compiler_barrier(); + + /* Save the thread's nesting count when it is not running. */ + THREAD->rcu.nesting_cnt = CPU->rcu.nesting_cnt; + /* Interrupt handlers might use RCU while idle in scheduler(). */ + CPU->rcu.nesting_cnt = 0; + + /* Preempted a reader critical section for the first time. */ + if (0 < THREAD->rcu.nesting_cnt && !THREAD->rcu.was_preempted) { + THREAD->rcu.was_preempted = true; + note_preempted_reader(); + } + + /* + * The preempted reader has been noted globally. There are therefore + * no readers running on this cpu so this is a quiescent state. + */ + _rcu_record_qs(); + + /* + * This cpu is holding up the current GP. Let the detector know + * it has just passed a quiescent state. + * + * The detector waits separately for preempted readers, so we have + * to notify the detector even if we have just preempted a reader. + */ + if (CPU->rcu.is_delaying_gp) { + CPU->rcu.is_delaying_gp = false; + semaphore_up(&rcu.remaining_readers); + } + + /* + * Forcefully associate the detector with the highest priority + * even if preempted due to its time slice running out. + * + * todo: Replace with strict scheduler priority classes. + */ + if (THREAD == rcu.detector_thr) { + THREAD->priority = -1; + } + else if (THREAD == CPU->rcu.reclaimer_thr) { + THREAD->priority = -1; + } + + upd_max_cbs_in_slice(); +} + +/** Called by the scheduler() when switching to a newly scheduled thread. */ +void rcu_before_thread_runs(void) +{ + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + ASSERT(0 == CPU->rcu.nesting_cnt); + + /* Load the thread's saved nesting count from before it was preempted. */ + CPU->rcu.nesting_cnt = THREAD->rcu.nesting_cnt; + /* + * In the unlikely event that a NMI occurs between the loading of the + * variables and setting signal_unlock, the NMI handler may invoke + * rcu_read_unlock() and clear signal_unlock. In that case we will + * incorrectly overwrite signal_unlock from false to true. This event + * situation benign and the next rcu_read_unlock() will at worst + * needlessly invoke _rcu_signal_unlock(). + */ + CPU->rcu.signal_unlock = THREAD->rcu.was_preempted || CPU->rcu.is_delaying_gp; +} + +/** Called from scheduler() when exiting the current thread. + * + * Preemption or interrupts are disabled and the scheduler() already + * switched away from the current thread, calling rcu_after_thread_ran(). + */ +void rcu_thread_exiting(void) +{ + ASSERT(THREAD != 0); + ASSERT(THREAD->state == Exiting); + ASSERT(PREEMPTION_DISABLED || interrupts_disabled()); + + /* + * The thread forgot to exit its reader critical section. + * It is a bug, but rather than letting the entire system lock up + * forcefully leave the reader section. The thread is not holding + * any references anyway since it is exiting so it is safe. + */ + if (0 < THREAD->rcu.nesting_cnt) { + THREAD->rcu.nesting_cnt = 1; + read_unlock_impl(&THREAD->rcu.nesting_cnt); + + printf("Bug: thread (id %" PRIu64 " \"%s\") exited while in RCU read" + " section.\n", THREAD->tid, THREAD->name); + } +} + + +#endif /* RCU_PREEMPT_PODZIMEK */ + +/** Announces the start of a new grace period for preexisting readers to ack. */ +static void start_new_gp(void) +{ + ASSERT(spinlock_locked(&rcu.gp_lock)); + + irq_spinlock_lock(&rcu.preempt_lock, true); + + /* Start a new GP. Announce to readers that a quiescent state is needed. */ + ++_rcu_cur_gp; + + /* + * Readers preempted before the start of this GP (next_preempted) + * are preexisting readers now that a GP started and will hold up + * the current GP until they exit their reader sections. + * + * Preempted readers from the previous GP have finished so + * cur_preempted is empty, but see comment in _rcu_record_qs(). + */ + list_concat(&rcu.cur_preempted, &rcu.next_preempted); + + irq_spinlock_unlock(&rcu.preempt_lock, true); +} + +/** Remove those cpus from the mask that have already passed a quiescent + * state since the start of the current grace period. + */ +static void rm_quiescent_cpus(cpu_mask_t *cpu_mask) +{ + /* + * Ensure the announcement of the start of a new GP (ie up-to-date + * cur_gp) propagates to cpus that are just coming out of idle + * mode before we sample their idle state flag. + * + * Cpus guarantee that after they set CPU->idle = true they will not + * execute any RCU reader sections without first setting idle to + * false and issuing a memory barrier. Therefore, if rm_quiescent_cpus() + * later on sees an idle cpu, but the cpu is just exiting its idle mode, + * the cpu must not have yet executed its memory barrier (otherwise + * it would pair up with this mem barrier and we would see idle == false). + * That memory barrier will pair up with the one below and ensure + * that a reader on the now-non-idle cpu will see the most current + * cur_gp. As a result, such a reader will never attempt to semaphore_up( + * pending_readers) during this GP, which allows the detector to + * ignore that cpu (the detector thinks it is idle). Moreover, any + * changes made by RCU updaters will have propagated to readers + * on the previously idle cpu -- again thanks to issuing a memory + * barrier after returning from idle mode. + * + * idle -> non-idle cpu | detector | reclaimer + * ------------------------------------------------------ + * rcu reader 1 | | rcu_call() + * MB X | | + * idle = true | | rcu_call() + * (no rcu readers allowed ) | | MB A in advance_cbs() + * MB Y | (...) | (...) + * (no rcu readers allowed) | | MB B in advance_cbs() + * idle = false | ++cur_gp | + * (no rcu readers allowed) | MB C | + * MB Z | signal gp_end | + * rcu reader 2 | | exec_cur_cbs() + * + * + * MB Y orders visibility of changes to idle for detector's sake. + * + * MB Z pairs up with MB C. The cpu making a transition from idle + * will see the most current value of cur_gp and will not attempt + * to notify the detector even if preempted during this GP. + * + * MB Z pairs up with MB A from the previous batch. Updaters' changes + * are visible to reader 2 even when the detector thinks the cpu is idle + * but it is not anymore. + * + * MB X pairs up with MB B. Late mem accesses of reader 1 are contained + * and visible before idling and before any callbacks are executed + * by reclaimers. + * + * In summary, the detector does not know of or wait for reader 2, but + * it does not have to since it is a new reader that will not access + * data from previous GPs and will see any changes. + */ + memory_barrier(); /* MB C */ + + cpu_mask_for_each(*cpu_mask, cpu_id) { + /* + * The cpu already checked for and passed through a quiescent + * state since the beginning of this GP. + * + * _rcu_cur_gp is modified by local detector thread only. + * Therefore, it is up-to-date even without a lock. + */ + bool cpu_acked_gp = (cpus[cpu_id].rcu.last_seen_gp == _rcu_cur_gp); + + /* + * Either the cpu is idle or it is exiting away from idle mode + * and already sees the most current _rcu_cur_gp. See comment + * in wait_for_readers(). + */ + bool cpu_idle = cpus[cpu_id].idle; + + if (cpu_acked_gp || cpu_idle) { + cpu_mask_reset(cpu_mask, cpu_id); + } + } +} + +/** Invokes sample_local_cpu(arg) on each cpu of reader_cpus. */ +static void sample_cpus(cpu_mask_t *reader_cpus, void *arg) +{ + const size_t max_conconcurrent_calls = 16; + smp_call_t call[max_conconcurrent_calls]; + size_t outstanding_calls = 0; + + cpu_mask_for_each(*reader_cpus, cpu_id) { + smp_call_async(cpu_id, sample_local_cpu, arg, &call[outstanding_calls]); + ++outstanding_calls; + + /* Update statistic. */ + if (CPU->id != cpu_id) + ++rcu.stat_smp_call_cnt; + + if (outstanding_calls == max_conconcurrent_calls) { + for (size_t k = 0; k < outstanding_calls; ++k) { + smp_call_wait(&call[k]); + } + + outstanding_calls = 0; + } + } + + for (size_t k = 0; k < outstanding_calls; ++k) { + smp_call_wait(&call[k]); + } +} + +static void upd_missed_gp_in_wait(rcu_gp_t completed_gp) +{ + ASSERT(CPU->rcu.cur_cbs_gp <= completed_gp); + + size_t delta = (size_t)(completed_gp - CPU->rcu.cur_cbs_gp); + CPU->rcu.stat_missed_gp_in_wait += delta; +} + +/** Globally note that the current thread was preempted in a reader section. */ +static void note_preempted_reader(void) +{ + irq_spinlock_lock(&rcu.preempt_lock, false); + + if (CPU->rcu.last_seen_gp != _rcu_cur_gp) { + /* The reader started before the GP started - we must wait for it.*/ + list_append(&THREAD->rcu.preempt_link, &rcu.cur_preempted); + } else { + /* + * The reader started after the GP started and this cpu + * already noted a quiescent state. We might block the next GP. + */ + list_append(&THREAD->rcu.preempt_link, &rcu.next_preempted); + } + + irq_spinlock_unlock(&rcu.preempt_lock, false); +} + +/** Remove the current thread from the global list of preempted readers. */ +static void rm_preempted_reader(void) +{ + irq_spinlock_lock(&rcu.preempt_lock, false); + + ASSERT(link_used(&THREAD->rcu.preempt_link)); + + bool prev_empty = list_empty(&rcu.cur_preempted); + list_remove(&THREAD->rcu.preempt_link); + bool now_empty = list_empty(&rcu.cur_preempted); + + /* This was the last reader in cur_preempted. */ + bool last_removed = now_empty && !prev_empty; + + /* + * Preempted readers are blocking the detector and + * this was the last reader blocking the current GP. + */ + if (last_removed && rcu.preempt_blocking_det) { + rcu.preempt_blocking_det = false; + semaphore_up(&rcu.remaining_readers); + } + + irq_spinlock_unlock(&rcu.preempt_lock, false); +} + +/** Waits for any preempted readers blocking this grace period to finish.*/ +static bool wait_for_preempt_reader(void) +{ + irq_spinlock_lock(&rcu.preempt_lock, true); + + bool reader_exists = !list_empty(&rcu.cur_preempted); + rcu.preempt_blocking_det = reader_exists; + + irq_spinlock_unlock(&rcu.preempt_lock, true); + + if (reader_exists) { + /* Update statistic. */ + ++rcu.stat_preempt_blocking_cnt; + + return semaphore_down_interruptable(&rcu.remaining_readers); + } + + return true; +} + +static void upd_max_cbs_in_slice(void) +{ + rcu_cpu_data_t *cr = &CPU->rcu; + + if (cr->arriving_cbs_cnt > cr->last_arriving_cnt) { + size_t arrived_cnt = cr->arriving_cbs_cnt - cr->last_arriving_cnt; + cr->stat_max_slice_cbs = max(arrived_cnt, cr->stat_max_slice_cbs); + } + + cr->last_arriving_cnt = cr->arriving_cbs_cnt; +} + +/** Prints RCU run-time statistics. */ +void rcu_print_stat(void) +{ + /* + * Don't take locks. Worst case is we get out-dated values. + * CPU local values are updated without any locks, so there + * are no locks to lock in order to get up-to-date values. + */ + +#ifdef RCU_PREEMPT_PODZIMEK + const char *algo = "podzimek-preempt-rcu"; +#elif defined(RCU_PREEMPT_A) + const char *algo = "a-preempt-rcu"; +#endif + + printf("Config: expedite_threshold=%d, critical_threshold=%d," + " detect_sleep=%dms, %s\n", + EXPEDITE_THRESHOLD, CRITICAL_THRESHOLD, DETECT_SLEEP_MS, algo); + printf("Completed GPs: %" PRIu64 "\n", rcu.completed_gp); + printf("Expedited GPs: %zu\n", rcu.stat_expedited_cnt); + printf("Delayed GPs: %zu (cpus w/ still running readers after gp sleep)\n", + rcu.stat_delayed_cnt); + printf("Preempt blocked GPs: %zu (waited for preempted readers; " + "running or not)\n", rcu.stat_preempt_blocking_cnt); + printf("Smp calls: %zu\n", rcu.stat_smp_call_cnt); + + printf("Max arrived callbacks per GP and CPU:\n"); + for (unsigned int i = 0; i < config.cpu_count; ++i) { + printf(" %zu", cpus[i].rcu.stat_max_cbs); + } + + printf("\nAvg arrived callbacks per GP and CPU (nonempty batches only):\n"); + for (unsigned int i = 0; i < config.cpu_count; ++i) { + printf(" %zu", cpus[i].rcu.stat_avg_cbs); + } + + printf("\nMax arrived callbacks per time slice and CPU:\n"); + for (unsigned int i = 0; i < config.cpu_count; ++i) { + printf(" %zu", cpus[i].rcu.stat_max_slice_cbs); + } + + printf("\nMissed GP notifications per CPU:\n"); + for (unsigned int i = 0; i < config.cpu_count; ++i) { + printf(" %zu", cpus[i].rcu.stat_missed_gps); + } + + printf("\nMissed GP notifications per CPU while waking up:\n"); + for (unsigned int i = 0; i < config.cpu_count; ++i) { + printf(" %zu", cpus[i].rcu.stat_missed_gp_in_wait); + } + printf("\n"); +} + +/** @} + */ Index: kernel/generic/src/synch/smc.c =================================================================== --- kernel/generic/src/synch/smc.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/synch/smc.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include sysarg_t sys_smc_coherence(uintptr_t va, size_t size) Index: kernel/generic/src/synch/spinlock.c =================================================================== --- kernel/generic/src/synch/spinlock.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/synch/spinlock.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -45,4 +45,5 @@ #include #include +#include #ifdef CONFIG_SMP Index: kernel/generic/src/synch/waitq.c =================================================================== --- kernel/generic/src/synch/waitq.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/synch/waitq.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -442,4 +442,42 @@ irq_spinlock_unlock(&wq->lock, true); } + +/** If there is a wakeup in progress actively waits for it to complete. + * + * The function returns once the concurrently running waitq_wakeup() + * exits. It returns immediately if there are no concurrent wakeups + * at the time. + * + * Example usage: + * @code + * void callback(waitq *wq) + * { + * // Do something and notify wait_for_completion() that we're done. + * waitq_wakeup(wq); + * } + * void wait_for_completion(void) + * { + * waitq wg; + * waitq_initialize(&wq); + * // Run callback() in the background, pass it wq. + * do_asynchronously(callback, &wq); + * // Wait for callback() to complete its work. + * waitq_sleep(&wq); + * // callback() completed its work, but it may still be accessing + * // wq in waitq_wakeup(). Therefore it is not yet safe to return + * // or it would clobber up our stack (where wq is stored). + * waitq_complete_wakeup(&wq); + * // waitq_wakeup() is complete, it is safe to free wq. + * } + * @endcode + * + * @param wq Pointer to a wait queue. + */ +void waitq_complete_wakeup(waitq_t *wq) +{ + irq_spinlock_lock(&wq->lock, true); + irq_spinlock_unlock(&wq->lock, true); +} + /** Internal SMP- and IRQ-unsafe version of waitq_wakeup() Index: kernel/generic/src/synch/workqueue.c =================================================================== --- kernel/generic/src/synch/workqueue.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/generic/src/synch/workqueue.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,976 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup generic + * @{ + */ + +/** + * @file + * @brief Work queue/thread pool that automatically adjusts its size + * depending on the current load. Queued work functions may sleep.. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WORKQ_MAGIC 0xf00c1333U +#define WORK_ITEM_MAGIC 0xfeec1777U + + +struct work_queue { + /* + * Protects everything except activate_worker. + * Must be acquired after any thread->locks. + */ + IRQ_SPINLOCK_DECLARE(lock); + + /* Activates a worker if new work arrives or if shutting down the queue. */ + condvar_t activate_worker; + + /* Queue of work_items ready to be dispatched. */ + list_t queue; + + /* List of worker threads. */ + list_t workers; + + /* Number of work items queued. */ + size_t item_cnt; + + /* Indicates the work queue is shutting down. */ + bool stopping; + const char *name; + + /* Total number of created worker threads. */ + size_t cur_worker_cnt; + /* Number of workers waiting for work to arrive. */ + size_t idle_worker_cnt; + /* Number of idle workers signaled that have not yet been woken up. */ + size_t activate_pending; + /* Number of blocked workers sleeping in work func() (ie not idle). */ + size_t blocked_worker_cnt; + + /* Number of pending signal_worker_op() operations. */ + size_t pending_op_cnt; + + link_t nb_link; + +#ifdef CONFIG_DEBUG + /* Magic cookie for integrity checks. Immutable. Accessed without lock. */ + uint32_t cookie; +#endif +}; + + +/** Min number of idle workers to keep. */ +static size_t min_worker_cnt; +/** Max total number of workers - be it blocked, idle, or active. */ +static size_t max_worker_cnt; +/** Max number of concurrently running active workers, ie not blocked nor idle. */ +static size_t max_concurrent_workers; +/** Max number of work items per active worker before a new worker is activated.*/ +static const size_t max_items_per_worker = 8; + +/** System wide work queue. */ +static struct work_queue g_work_queue; + +static int booting = true; + + +typedef struct { + IRQ_SPINLOCK_DECLARE(lock); + condvar_t req_cv; + thread_t *thread; + list_t work_queues; +} nonblock_adder_t; + +static nonblock_adder_t nonblock_adder; + + + +/** Typedef a worker thread signaling operation prototype. */ +typedef void (*signal_op_t)(struct work_queue *workq); + + +/* Fwd decl. */ +static void workq_preinit(struct work_queue *workq, const char *name); +static bool add_worker(struct work_queue *workq); +static void interrupt_workers(struct work_queue *workq); +static void wait_for_workers(struct work_queue *workq); +static int _workq_enqueue(struct work_queue *workq, work_t *work_item, + work_func_t func, bool can_block); +static void init_work_item(work_t *work_item, work_func_t func); +static signal_op_t signal_worker_logic(struct work_queue *workq, bool can_block); +static void worker_thread(void *arg); +static bool dequeue_work(struct work_queue *workq, work_t **pwork_item); +static bool worker_unnecessary(struct work_queue *workq); +static void cv_wait(struct work_queue *workq); +static void nonblock_init(void); +static bool workq_corrupted(struct work_queue *workq); +static bool work_item_corrupted(work_t *work_item); + + +/** Creates worker thread for the system-wide worker queue. */ +void workq_global_worker_init(void) +{ + /* + * No need for additional synchronization. Stores to word-sized + * variables are atomic and the change will eventually propagate. + * Moreover add_worker() includes the necessary memory barriers + * in spinlock lock/unlock(). + */ + booting = false; + + nonblock_init(); + + if (!add_worker(&g_work_queue)) + panic("Could not create a single global work queue worker!\n"); + +} + +/** Initializes the system wide work queue and support for other work queues. */ +void workq_global_init(void) +{ + /* Keep idle workers on 1/4-th of cpus, but at least 2 threads. */ + min_worker_cnt = max(2, config.cpu_count / 4); + /* Allow max 8 sleeping work items per cpu. */ + max_worker_cnt = max(32, 8 * config.cpu_count); + /* Maximum concurrency without slowing down the system. */ + max_concurrent_workers = max(2, config.cpu_count); + + workq_preinit(&g_work_queue, "kworkq"); +} + +/** Stops the system global work queue and waits for all work items to complete.*/ +void workq_global_stop(void) +{ + workq_stop(&g_work_queue); +} + +/** Creates and initializes a work queue. Returns NULL upon failure. */ +struct work_queue * workq_create(const char *name) +{ + struct work_queue *workq = malloc(sizeof(struct work_queue), 0); + + if (workq) { + if (workq_init(workq, name)) { + ASSERT(!workq_corrupted(workq)); + return workq; + } + + free(workq); + } + + return NULL; +} + +/** Frees work queue resources and stops it if it had not been done so already.*/ +void workq_destroy(struct work_queue *workq) +{ + ASSERT(!workq_corrupted(workq)); + + irq_spinlock_lock(&workq->lock, true); + bool stopped = workq->stopping; + size_t running_workers = workq->cur_worker_cnt; + irq_spinlock_unlock(&workq->lock, true); + + if (!stopped) { + workq_stop(workq); + } else { + ASSERT(0 == running_workers); + } + +#ifdef CONFIG_DEBUG + workq->cookie = 0; +#endif + + free(workq); +} + +/** Initializes workq structure without creating any workers. */ +static void workq_preinit(struct work_queue *workq, const char *name) +{ +#ifdef CONFIG_DEBUG + workq->cookie = WORKQ_MAGIC; +#endif + + irq_spinlock_initialize(&workq->lock, name); + condvar_initialize(&workq->activate_worker); + + list_initialize(&workq->queue); + list_initialize(&workq->workers); + + workq->item_cnt = 0; + workq->stopping = false; + workq->name = name; + + workq->cur_worker_cnt = 1; + workq->idle_worker_cnt = 0; + workq->activate_pending = 0; + workq->blocked_worker_cnt = 0; + + workq->pending_op_cnt = 0; + link_initialize(&workq->nb_link); +} + +/** Initializes a work queue. Returns true if successful. + * + * Before destroying a work queue it must be stopped via + * workq_stop(). + */ +int workq_init(struct work_queue *workq, const char *name) +{ + workq_preinit(workq, name); + return add_worker(workq); +} + +/** Add a new worker thread. Returns false if the thread could not be created. */ +static bool add_worker(struct work_queue *workq) +{ + ASSERT(!workq_corrupted(workq)); + + thread_t *thread = thread_create(worker_thread, workq, TASK, + THREAD_FLAG_NONE, workq->name); + + if (!thread) { + irq_spinlock_lock(&workq->lock, true); + + /* cur_worker_cnt proactively increased in signal_worker_logic() .*/ + ASSERT(0 < workq->cur_worker_cnt); + --workq->cur_worker_cnt; + + irq_spinlock_unlock(&workq->lock, true); + return false; + } + + /* Respect lock ordering. */ + irq_spinlock_lock(&thread->lock, true); + irq_spinlock_lock(&workq->lock, false); + + bool success; + + if (!workq->stopping) { + success = true; + + /* Try to distribute workers among cpus right away. */ + unsigned int cpu_id = (workq->cur_worker_cnt) % config.cpu_active; + + if (!cpus[cpu_id].active) + cpu_id = CPU->id; + + thread->workq = workq; + thread->cpu = &cpus[cpu_id]; + thread->workq_blocked = false; + thread->workq_idling = false; + link_initialize(&thread->workq_link); + + list_append(&thread->workq_link, &workq->workers); + } else { + /* + * Work queue is shutting down - we must not add the worker + * and we cannot destroy it without ready-ing it. Mark it + * interrupted so the worker exits right away without even + * touching workq. + */ + success = false; + + /* cur_worker_cnt proactively increased in signal_worker() .*/ + ASSERT(0 < workq->cur_worker_cnt); + --workq->cur_worker_cnt; + } + + irq_spinlock_unlock(&workq->lock, false); + irq_spinlock_unlock(&thread->lock, true); + + if (!success) { + thread_interrupt(thread); + } + + thread_ready(thread); + + return success; +} + +/** Shuts down the work queue. Waits for all pending work items to complete. + * + * workq_stop() may only be run once. + */ +void workq_stop(struct work_queue *workq) +{ + ASSERT(!workq_corrupted(workq)); + + interrupt_workers(workq); + wait_for_workers(workq); +} + +/** Notifies worker threads the work queue is shutting down. */ +static void interrupt_workers(struct work_queue *workq) +{ + irq_spinlock_lock(&workq->lock, true); + + /* workq_stop() may only be called once. */ + ASSERT(!workq->stopping); + workq->stopping = true; + + /* Respect lock ordering - do not hold workq->lock during broadcast. */ + irq_spinlock_unlock(&workq->lock, true); + + condvar_broadcast(&workq->activate_worker); +} + +/** Waits for all worker threads to exit. */ +static void wait_for_workers(struct work_queue *workq) +{ + ASSERT(!PREEMPTION_DISABLED); + + irq_spinlock_lock(&workq->lock, true); + + list_foreach_safe(workq->workers, cur_worker, next_worker) { + thread_t *worker = list_get_instance(cur_worker, thread_t, workq_link); + list_remove(cur_worker); + + /* Wait without the lock. */ + irq_spinlock_unlock(&workq->lock, true); + + thread_join(worker); + thread_detach(worker); + + irq_spinlock_lock(&workq->lock, true); + } + + ASSERT(list_empty(&workq->workers)); + + /* Wait for deferred add_worker_op(), signal_worker_op() to finish. */ + while (0 < workq->cur_worker_cnt || 0 < workq->pending_op_cnt) { + irq_spinlock_unlock(&workq->lock, true); + + scheduler(); + + irq_spinlock_lock(&workq->lock, true); + } + + irq_spinlock_unlock(&workq->lock, true); +} + +/** Queues a function into the global wait queue without blocking. + * + * See workq_enqueue_noblock() for more details. + */ +int workq_global_enqueue_noblock(work_t *work_item, work_func_t func) +{ + return workq_enqueue_noblock(&g_work_queue, work_item, func); +} + +/** Queues a function into the global wait queue; may block. + * + * See workq_enqueue() for more details. + */ +int workq_global_enqueue(work_t *work_item, work_func_t func) +{ + return workq_enqueue(&g_work_queue, work_item, func); +} + +/** Adds a function to be invoked in a separate thread without blocking. + * + * workq_enqueue_noblock() is guaranteed not to block. It is safe + * to invoke from interrupt handlers. + * + * Consider using workq_enqueue() instead if at all possible. Otherwise, + * your work item may have to wait for previously enqueued sleeping + * work items to complete if you are unlucky. + * + * @param workq Work queue where to queue the work item. + * @param work_item Work item bookkeeping structure. Must be valid + * until func() is entered. + * @param func User supplied function to invoke in a worker thread. + + * @return false if work queue is shutting down; function is not + * queued for further processing. + * @return true Otherwise. func() will be invoked in a separate thread. + */ +int workq_enqueue_noblock(struct work_queue *workq, work_t *work_item, + work_func_t func) +{ + return _workq_enqueue(workq, work_item, func, false); +} + +/** Adds a function to be invoked in a separate thread; may blocking. + * + * While the workq_enqueue() is unlikely to block, it may do so if too + * many previous work items blocked sleeping. + * + * @param workq Work queue where to queue the work item. + * @param work_item Work item bookkeeping structure. Must be valid + * until func() is entered. + * @param func User supplied function to invoke in a worker thread. + + * @return false if work queue is shutting down; function is not + * queued for further processing. + * @return true Otherwise. func() will be invoked in a separate thread. + */ +int workq_enqueue(struct work_queue *workq, work_t *work_item, work_func_t func) +{ + return _workq_enqueue(workq, work_item, func, true); +} + +/** Adds a work item that will be processed by a separate worker thread. + * + * func() will be invoked in another kernel thread and may block. + * + * Prefer to call _workq_enqueue() with can_block set. Otherwise + * your work item may have to wait for sleeping work items to complete. + * If all worker threads are blocked/sleeping a new worker thread cannot + * be create without can_block set because creating a thread might + * block due to low memory conditions. + * + * @param workq Work queue where to queue the work item. + * @param work_item Work item bookkeeping structure. Must be valid + * until func() is entered. + * @param func User supplied function to invoke in a worker thread. + * @param can_block May adding this work item block? + + * @return false if work queue is shutting down; function is not + * queued for further processing. + * @return true Otherwise. + */ +static int _workq_enqueue(struct work_queue *workq, work_t *work_item, + work_func_t func, bool can_block) +{ + ASSERT(!workq_corrupted(workq)); + + bool success = true; + signal_op_t signal_op = NULL; + + irq_spinlock_lock(&workq->lock, true); + + if (workq->stopping) { + success = false; + } else { + init_work_item(work_item, func); + list_append(&work_item->queue_link, &workq->queue); + ++workq->item_cnt; + success = true; + + if (!booting) { + signal_op = signal_worker_logic(workq, can_block); + } else { + /* + * During boot there are no workers to signal. Just queue + * the work and let future workers take care of it. + */ + } + } + + irq_spinlock_unlock(&workq->lock, true); + + if (signal_op) { + signal_op(workq); + } + + return success; +} + +/** Prepare an item to be added to the work item queue. */ +static void init_work_item(work_t *work_item, work_func_t func) +{ +#ifdef CONFIG_DEBUG + work_item->cookie = WORK_ITEM_MAGIC; +#endif + + link_initialize(&work_item->queue_link); + work_item->func = func; +} + +/** Returns the number of workers running work func() that are not blocked. */ +static size_t active_workers_now(struct work_queue *workq) +{ + ASSERT(irq_spinlock_locked(&workq->lock)); + + /* Workers blocked are sleeping in the work function (ie not idle). */ + ASSERT(workq->blocked_worker_cnt <= workq->cur_worker_cnt); + /* Idle workers are waiting for more work to arrive in condvar_wait. */ + ASSERT(workq->idle_worker_cnt <= workq->cur_worker_cnt); + + /* Idle + blocked workers == sleeping worker threads. */ + size_t sleeping_workers = workq->blocked_worker_cnt + workq->idle_worker_cnt; + + ASSERT(sleeping_workers <= workq->cur_worker_cnt); + /* Workers pending activation are idle workers not yet given a time slice. */ + ASSERT(workq->activate_pending <= workq->idle_worker_cnt); + + /* + * Workers actively running the work func() this very moment and + * are neither blocked nor idle. Exclude ->activate_pending workers + * since they will run their work func() once they get a time slice + * and are not running it right now. + */ + return workq->cur_worker_cnt - sleeping_workers; +} + +/** + * Returns the number of workers that are running or are about to run work + * func() and that are not blocked. + */ +static size_t active_workers(struct work_queue *workq) +{ + ASSERT(irq_spinlock_locked(&workq->lock)); + + /* + * Workers actively running the work func() and are neither blocked nor + * idle. ->activate_pending workers will run their work func() once they + * get a time slice after waking from a condvar wait, so count them + * as well. + */ + return active_workers_now(workq) + workq->activate_pending; +} + +static void add_worker_noblock_op(struct work_queue *workq) +{ + condvar_signal(&nonblock_adder.req_cv); +} + +static void add_worker_op(struct work_queue *workq) +{ + add_worker(workq); +} + +static void signal_worker_op(struct work_queue *workq) +{ + ASSERT(!workq_corrupted(workq)); + + condvar_signal(&workq->activate_worker); + + irq_spinlock_lock(&workq->lock, true); + ASSERT(0 < workq->pending_op_cnt); + --workq->pending_op_cnt; + irq_spinlock_unlock(&workq->lock, true); +} + +/** Determines how to signal workers if at all. + * + * @param workq Work queue where a new work item was queued. + * @param can_block True if we may block while signaling a worker or creating + * a new worker. + * + * @return Function that will notify workers or NULL if no action is needed. + */ +static signal_op_t signal_worker_logic(struct work_queue *workq, bool can_block) +{ + ASSERT(!workq_corrupted(workq)); + ASSERT(irq_spinlock_locked(&workq->lock)); + + /* Only signal workers if really necessary. */ + signal_op_t signal_op = NULL; + + /* + * Workers actively running the work func() and neither blocked nor idle. + * Including ->activate_pending workers that will run their work func() + * once they get a time slice. + */ + size_t active = active_workers(workq); + /* Max total allowed number of work items queued for active workers. */ + size_t max_load = active * max_items_per_worker; + + /* Active workers are getting overwhelmed - activate another. */ + if (max_load < workq->item_cnt) { + + size_t remaining_idle = + workq->idle_worker_cnt - workq->activate_pending; + + /* Idle workers still exist - activate one. */ + if (remaining_idle > 0) { + /* + * Directly changing idle_worker_cnt here would not allow + * workers to recognize spurious wake-ups. Change + * activate_pending instead. + */ + ++workq->activate_pending; + ++workq->pending_op_cnt; + signal_op = signal_worker_op; + } else { + /* No idle workers remain. Request that a new one be created. */ + bool need_worker = (active < max_concurrent_workers) + && (workq->cur_worker_cnt < max_worker_cnt); + + if (need_worker && can_block) { + signal_op = add_worker_op; + /* + * It may take some time to actually create the worker. + * We don't want to swamp the thread pool with superfluous + * worker creation requests so pretend it was already + * created and proactively increase the worker count. + */ + ++workq->cur_worker_cnt; + } + + /* + * We cannot create a new worker but we need one desperately + * because all workers are blocked in their work functions. + */ + if (need_worker && !can_block && 0 == active) { + ASSERT(0 == workq->idle_worker_cnt); + + irq_spinlock_lock(&nonblock_adder.lock, true); + + if (nonblock_adder.thread && !link_used(&workq->nb_link)) { + signal_op = add_worker_noblock_op; + ++workq->cur_worker_cnt; + list_append(&workq->nb_link, &nonblock_adder.work_queues); + } + + irq_spinlock_unlock(&nonblock_adder.lock, true); + } + } + } else { + /* + * There are enough active/running workers to process the queue. + * No need to signal/activate any new workers. + */ + signal_op = NULL; + } + + return signal_op; +} + +/** Executes queued work items. */ +static void worker_thread(void *arg) +{ + /* + * The thread has been created after the work queue was ordered to stop. + * Do not access the work queue and return immediately. + */ + if (thread_interrupted(THREAD)) { + thread_detach(THREAD); + return; + } + + ASSERT(arg != NULL); + + struct work_queue *workq = arg; + work_t *work_item; + + while (dequeue_work(workq, &work_item)) { + /* Copy the func field so func() can safely free work_item. */ + work_func_t func = work_item->func; + + func(work_item); + } +} + +/** Waits and retrieves a work item. Returns false if the worker should exit. */ +static bool dequeue_work(struct work_queue *workq, work_t **pwork_item) +{ + ASSERT(!workq_corrupted(workq)); + + irq_spinlock_lock(&workq->lock, true); + + /* Check if we should exit if load is low. */ + if (!workq->stopping && worker_unnecessary(workq)) { + /* There are too many workers for this load. Exit. */ + ASSERT(0 < workq->cur_worker_cnt); + --workq->cur_worker_cnt; + list_remove(&THREAD->workq_link); + irq_spinlock_unlock(&workq->lock, true); + + thread_detach(THREAD); + return false; + } + + bool stop = false; + + /* Wait for work to arrive. */ + while (list_empty(&workq->queue) && !workq->stopping) { + cv_wait(workq); + + if (0 < workq->activate_pending) + --workq->activate_pending; + } + + /* Process remaining work even if requested to stop. */ + if (!list_empty(&workq->queue)) { + link_t *work_link = list_first(&workq->queue); + *pwork_item = list_get_instance(work_link, work_t, queue_link); + +#ifdef CONFIG_DEBUG + ASSERT(!work_item_corrupted(*pwork_item)); + (*pwork_item)->cookie = 0; +#endif + list_remove(work_link); + --workq->item_cnt; + + stop = false; + } else { + /* Requested to stop and no more work queued. */ + ASSERT(workq->stopping); + --workq->cur_worker_cnt; + stop = true; + } + + irq_spinlock_unlock(&workq->lock, true); + + return !stop; +} + +/** Returns true if for the given load there are too many workers. */ +static bool worker_unnecessary(struct work_queue *workq) +{ + ASSERT(irq_spinlock_locked(&workq->lock)); + + /* No work is pending. We don't need too many idle threads. */ + if (list_empty(&workq->queue)) { + /* There are too many idle workers. Exit. */ + return (min_worker_cnt <= workq->idle_worker_cnt); + } else { + /* + * There is work but we are swamped with too many active workers + * that were woken up from sleep at around the same time. We + * don't need another worker fighting for cpu time. + */ + size_t active = active_workers_now(workq); + return (max_concurrent_workers < active); + } +} + +/** Waits for a signal to activate_worker. Thread marked idle while waiting. */ +static void cv_wait(struct work_queue *workq) +{ + ++workq->idle_worker_cnt; + THREAD->workq_idling = true; + + /* Ignore lock ordering just here. */ + ASSERT(irq_spinlock_locked(&workq->lock)); + + _condvar_wait_timeout_irq_spinlock(&workq->activate_worker, + &workq->lock, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE); + + ASSERT(!workq_corrupted(workq)); + ASSERT(irq_spinlock_locked(&workq->lock)); + + THREAD->workq_idling = false; + --workq->idle_worker_cnt; +} + + +/** Invoked from thread_ready() right before the thread is woken up. */ +void workq_before_thread_is_ready(thread_t *thread) +{ + ASSERT(thread); + ASSERT(irq_spinlock_locked(&thread->lock)); + + /* Worker's work func() is about to wake up from sleeping. */ + if (thread->workq && thread->workq_blocked) { + /* Must be blocked in user work func() and not be waiting for work. */ + ASSERT(!thread->workq_idling); + ASSERT(thread->state == Sleeping); + ASSERT(THREAD != thread); + ASSERT(!workq_corrupted(thread->workq)); + + /* Protected by thread->lock */ + thread->workq_blocked = false; + + irq_spinlock_lock(&thread->workq->lock, true); + --thread->workq->blocked_worker_cnt; + irq_spinlock_unlock(&thread->workq->lock, true); + } +} + +/** Invoked from scheduler() before switching away from a thread. */ +void workq_after_thread_ran(void) +{ + ASSERT(THREAD); + ASSERT(irq_spinlock_locked(&THREAD->lock)); + + /* Worker's work func() is about to sleep/block. */ + if (THREAD->workq && THREAD->state == Sleeping && !THREAD->workq_idling) { + ASSERT(!THREAD->workq_blocked); + ASSERT(!workq_corrupted(THREAD->workq)); + + THREAD->workq_blocked = true; + + irq_spinlock_lock(&THREAD->workq->lock, false); + + ++THREAD->workq->blocked_worker_cnt; + + bool can_block = false; + signal_op_t op = signal_worker_logic(THREAD->workq, can_block); + + irq_spinlock_unlock(&THREAD->workq->lock, false); + + if (op) { + ASSERT(add_worker_noblock_op == op || signal_worker_op == op); + op(THREAD->workq); + } + } +} + +/** Prints stats of the work queue to the kernel console. */ +void workq_print_info(struct work_queue *workq) +{ + irq_spinlock_lock(&workq->lock, true); + + size_t total = workq->cur_worker_cnt; + size_t blocked = workq->blocked_worker_cnt; + size_t idle = workq->idle_worker_cnt; + size_t active = active_workers(workq); + size_t items = workq->item_cnt; + bool stopping = workq->stopping; + bool worker_surplus = worker_unnecessary(workq); + const char *load_str = worker_surplus ? "decreasing" : + (0 < workq->activate_pending) ? "increasing" : "stable"; + + irq_spinlock_unlock(&workq->lock, true); + + printf( + "Configuration: max_worker_cnt=%zu, min_worker_cnt=%zu,\n" + " max_concurrent_workers=%zu, max_items_per_worker=%zu\n" + "Workers: %zu\n" + "Active: %zu (workers currently processing work)\n" + "Blocked: %zu (work functions sleeping/blocked)\n" + "Idle: %zu (idle workers waiting for more work)\n" + "Items: %zu (queued not yet dispatched work)\n" + "Stopping: %d\n" + "Load: %s\n", + max_worker_cnt, min_worker_cnt, + max_concurrent_workers, max_items_per_worker, + total, + active, + blocked, + idle, + items, + stopping, + load_str + ); +} + +/** Prints stats of the global work queue. */ +void workq_global_print_info(void) +{ + workq_print_info(&g_work_queue); +} + + +static bool dequeue_add_req(nonblock_adder_t *info, struct work_queue **pworkq) +{ + bool stop = false; + + irq_spinlock_lock(&info->lock, true); + + while (list_empty(&info->work_queues) && !stop) { + int ret = _condvar_wait_timeout_irq_spinlock(&info->req_cv, + &info->lock, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_INTERRUPTIBLE); + + stop = (ret == ESYNCH_INTERRUPTED); + } + + if (!stop) { + *pworkq = list_get_instance(list_first(&info->work_queues), + struct work_queue, nb_link); + + ASSERT(!workq_corrupted(*pworkq)); + + list_remove(&(*pworkq)->nb_link); + } + + irq_spinlock_unlock(&info->lock, true); + + return !stop; +} + +static void thr_nonblock_add_worker(void *arg) +{ + nonblock_adder_t *info = arg; + struct work_queue *workq; + + while (dequeue_add_req(info, &workq)) { + add_worker(workq); + } +} + + +static void nonblock_init(void) +{ + irq_spinlock_initialize(&nonblock_adder.lock, "kworkq-nb.lock"); + condvar_initialize(&nonblock_adder.req_cv); + list_initialize(&nonblock_adder.work_queues); + + nonblock_adder.thread = thread_create(thr_nonblock_add_worker, + &nonblock_adder, TASK, THREAD_FLAG_NONE, "kworkq-nb"); + + if (nonblock_adder.thread) { + thread_ready(nonblock_adder.thread); + } else { + /* + * We won't be able to add workers without blocking if all workers + * sleep, but at least boot the system. + */ + printf("Failed to create kworkq-nb. Sleeping work may stall the workq.\n"); + } +} + +/** Returns true if the workq is definitely corrupted; false if not sure. + * + * Can be used outside of any locks. + */ +static bool workq_corrupted(struct work_queue *workq) +{ +#ifdef CONFIG_DEBUG + /* + * Needed to make the most current cookie value set by workq_preinit() + * visible even if we access the workq right after it is created but + * on a different cpu. Otherwise, workq_corrupted() would not work + * outside a lock. + */ + memory_barrier(); + return NULL == workq || workq->cookie != WORKQ_MAGIC; +#else + return false; +#endif +} + +/** Returns true if the work_item is definitely corrupted; false if not sure. + * + * Must be used with the work queue protecting spinlock locked. + */ +static bool work_item_corrupted(work_t *work_item) +{ +#ifdef CONFIG_DEBUG + return NULL == work_item || work_item->cookie != WORK_ITEM_MAGIC; +#else + return false; +#endif +} + +/** @} + */ Index: kernel/generic/src/time/clock.c =================================================================== --- kernel/generic/src/time/clock.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/time/clock.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -212,5 +212,5 @@ irq_spinlock_unlock(&THREAD->lock, false); - if ((!ticks) && (!PREEMPTION_DISABLED)) { + if (ticks == 0 && PREEMPTION_ENABLED) { scheduler(); #ifdef CONFIG_UDEBUG Index: kernel/generic/src/udebug/udebug.c =================================================================== --- kernel/generic/src/udebug/udebug.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/generic/src/udebug/udebug.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -44,4 +44,6 @@ #include #include +#include +#include /** Initialize udebug part of task structure. Index: kernel/test/atomic/atomic1.c =================================================================== --- kernel/test/atomic/atomic1.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/test/atomic/atomic1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -60,4 +60,21 @@ return "Failed atomic_get() after atomic_predec()"; + void *ptr = 0; + void *a_ptr = &a; + if (atomic_cas_ptr(&ptr, 0, a_ptr) != 0) + return "Failed atomic_cas_ptr(): bad return value"; + if (ptr != a_ptr) + return "Failed atomic_cas_ptr(): bad pointer value"; + if (atomic_cas_ptr(&ptr, 0, 0) != a_ptr) + return "Failed atomic_cas_ptr(): indicated change"; + if (ptr != a_ptr) + return "Failed atomic_cas_ptr(): changed the ptr"; + + ptr = 0; + if (atomic_set_return_ptr(&ptr, a_ptr) != 0) + return "Failed atomic_set_return_ptr()"; + if (atomic_set_return_ptr_local(&ptr, 0) != a_ptr || ptr != 0) + return "Failed atomic_set_return_ptr_local()"; + return NULL; } Index: kernel/test/cht/cht1.c =================================================================== --- kernel/test/cht/cht1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/cht/cht1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,560 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +typedef struct val { + /* Place at the top to simplify re-casting. */ + cht_link_t link; + size_t hash; + size_t unique_id; + bool deleted; + bool mark; +} val_t; + +static size_t val_hash(const cht_link_t *item) +{ + val_t *v = member_to_inst(item, val_t, link); + ASSERT(v->hash == (v->unique_id % 10)); + return v->hash; +} + +static size_t val_key_hash(void *key) +{ + return (uintptr_t)key % 10; +} + +static bool val_equal(const cht_link_t *item1, const cht_link_t *item2) +{ + val_t *v1 = member_to_inst(item1, val_t, link); + val_t *v2 = member_to_inst(item2, val_t, link); + return v1->unique_id == v2->unique_id; +} + +static bool val_key_equal(void *key, const cht_link_t *item2) +{ + val_t *v2 = member_to_inst(item2, val_t, link); + return (uintptr_t)key == v2->unique_id; +} + +static void val_rm_callback(cht_link_t *item) +{ + val_t *v = member_to_inst(item, val_t, link); + ASSERT(!v->deleted); + v->deleted = true; + free(v); +} + + +static cht_ops_t val_ops = { + .hash = val_hash, + .key_hash = val_key_hash, + .equal = val_equal, + .key_equal = val_key_equal, + .remove_callback = val_rm_callback, +}; + +static void set_val(val_t *v, size_t h, size_t uid) +{ + v->hash = h; + v->unique_id = uid; + v->deleted = false; + v->mark = false; +} + +/*-------------------------------------------------------------------*/ + + +static const char * do_sanity_test(cht_t *h) +{ + if (cht_find_lazy(h, 0)) + return "Found lazy in empty table."; + + if (cht_find(h, 0)) + return "Found in empty table."; + + if (cht_remove_key(h, 0)) + return "Removed from empty table."; + + const int val_cnt = 6; + val_t *v[6] = {0}; + + for (int i = 0; i < val_cnt; ++i) + v[i] = malloc(sizeof(val_t), 0); + + size_t key[] = { 1, 1, 1, 11, 12, 13 }; + + /* First three are identical */ + for (int i = 0; i < 3; ++i) + set_val(v[i], 1, key[i]); + + /* Same hash, different key.*/ + set_val(v[3], 1, key[3]); + + /* Different hashes and keys. */ + set_val(v[4], 2, key[4]); + set_val(v[5], 3, key[5]); + + if (!cht_insert_unique(h, &v[0]->link)) + return "Duplicates in empty"; + + if (cht_insert_unique(h, &v[1]->link)) + return "Inserted a duplicate"; + + if (!cht_insert_unique(h, &v[3]->link)) + return "Refused non-equal item but with a hash in table."; + + cht_insert(h, &v[1]->link); + cht_insert(h, &v[2]->link); + + bool ok = true; + ok = ok && cht_insert_unique(h, &v[4]->link); + ok = ok && cht_insert_unique(h, &v[5]->link); + + if (!ok) + return "Refused unique ins 4, 5."; + + if (cht_find(h, (void*)0)) + return "Fantom find."; + + cht_link_t *item = cht_find(h, (void*)v[5]->unique_id); + if (!item || item != &v[5]->link) + return "Missing 5."; + + item = cht_find_next(h, &v[5]->link); + if (item) + return "Found nonexisting duplicate 5"; + + item = cht_find(h, (void*)v[3]->unique_id); + if (!item || item != &v[3]->link) + return "Missing 3."; + + item = cht_find_next(h, &v[3]->link); + if (item) + return "Found nonexisting duplicate 3, same hash as others."; + + item = cht_find(h, (void*)v[0]->unique_id); + ((val_t*)item)->mark = true; + + for (int k = 1; k < 3; ++k) { + item = cht_find_next(h, item); + if (!item) + return "Did not find an inserted duplicate"; + + val_t *val = ((val_t*)item); + + if (val->unique_id != v[0]->unique_id) + return "Found item with a different key."; + if (val->mark) + return "Found twice the same node."; + val->mark = true; + } + + for (int i = 0; i < 3; ++i) { + if (!v[i]->mark) + return "Did not find all duplicates"; + + v[i]->mark = false; + } + + if (cht_find_next(h, item)) + return "Found non-existing duplicate."; + + item = cht_find_next(h, cht_find(h, (void*)key[0])); + + ((val_t*)item)->mark = true; + if (!cht_remove_item(h, item)) + return "Failed to remove inserted item"; + + item = cht_find(h, (void*)key[0]); + if (!item || ((val_t*)item)->mark) + return "Did not find proper item."; + + item = cht_find_next(h, item); + if (!item || ((val_t*)item)->mark) + return "Did not find proper duplicate."; + + item = cht_find_next(h, item); + if (item) + return "Found removed duplicate"; + + if (2 != cht_remove_key(h, (void*)key[0])) + return "Failed to remove all duplicates"; + + if (cht_find(h, (void*)key[0])) + return "Found removed key"; + + if (!cht_find(h, (void*)key[3])) + return "Removed incorrect key"; + + for (size_t k = 0; k < sizeof(v) / sizeof(v[0]); ++k) { + cht_remove_key(h, (void*)key[k]); + } + + for (size_t k = 0; k < sizeof(v) / sizeof(v[0]); ++k) { + if (cht_find(h, (void*)key[k])) + return "Found a key in a cleared table"; + } + + return 0; +} + +static const char * sanity_test(void) +{ + cht_t h; + if (!cht_create(&h, 5, 0, 0, &val_ops)) + return "Could not create the table."; + + rcu_read_lock(); + const char *err = do_sanity_test(&h); + rcu_read_unlock(); + + cht_destroy(&h); + + return err; +} + +/*-------------------------------------------------------------------*/ + +static size_t next_rand(size_t seed) +{ + return (seed * 1103515245 + 12345) & ((1U << 31) - 1); +} + +/*-------------------------------------------------------------------*/ +typedef struct { + cht_link_t link; + size_t key; + bool free; + bool inserted; + bool deleted; +} stress_t; + +typedef struct { + cht_t *h; + int *stop; + stress_t *elem; + size_t elem_cnt; + size_t upd_prob; + size_t wave_cnt; + size_t wave_elems; + size_t id; + bool failed; +} stress_work_t; + +static size_t stress_hash(const cht_link_t *item) +{ + return ((stress_t*)item)->key >> 8; +} +static size_t stress_key_hash(void *key) +{ + return ((size_t)key) >> 8; +} +static bool stress_equal(const cht_link_t *item1, const cht_link_t *item2) +{ + return ((stress_t*)item1)->key == ((stress_t*)item2)->key; +} +static bool stress_key_equal(void *key, const cht_link_t *item) +{ + return ((size_t)key) == ((stress_t*)item)->key; +} +static void stress_rm_callback(cht_link_t *item) +{ + if (((stress_t*)item)->free) + free(item); + else + ((stress_t*)item)->deleted = true; +} + +cht_ops_t stress_ops = { + .hash = stress_hash, + .key_hash = stress_key_hash, + .equal = stress_equal, + .key_equal = stress_key_equal, + .remove_callback = stress_rm_callback +}; + +static void resize_stresser(void *arg) +{ + stress_work_t *work = (stress_work_t *)arg; + + for (size_t k = 0; k < work->wave_cnt; ++k) { + TPRINTF("I{"); + for (size_t i = 0; i < work->wave_elems; ++i) { + stress_t *s = malloc(sizeof(stress_t), FRAME_ATOMIC); + if (!s) { + TPRINTF("[out-of-mem]\n"); + goto out_of_mem; + } + + s->free = true; + s->key = (i << 8) + work->id; + + cht_insert(work->h, &s->link); + } + TPRINTF("}"); + + thread_sleep(2); + + TPRINTF("R<"); + for (size_t i = 0; i < work->wave_elems; ++i) { + size_t key = (i << 8) + work->id; + + if (1 != cht_remove_key(work->h, (void*)key)) { + TPRINTF("Err: Failed to remove inserted item\n"); + goto failed; + } + } + TPRINTF(">"); + } + + /* Request that others stop. */ + *work->stop = 1; + return; + +failed: + work->failed = true; + +out_of_mem: + /* Request that others stop. */ + *work->stop = 1; + + /* Remove anything we may have inserted. */ + for (size_t i = 0; i < work->wave_elems; ++i) { + size_t key = (i << 8) + work->id; + cht_remove_key(work->h, (void*)key); + } +} + +static void op_stresser(void *arg) +{ + stress_work_t *work = (stress_work_t *)arg; + ASSERT(0 == *work->stop); + + size_t loops = 0; + size_t seed = work->id; + + while (0 == *work->stop && !work->failed) { + seed = next_rand(seed); + bool upd = ((seed % 100) <= work->upd_prob); + seed = next_rand(seed); + size_t elem_idx = seed % work->elem_cnt; + + ++loops; + if (0 == loops % (1024 * 1024)) { + /* Make the most current work->stop visible. */ + read_barrier(); + TPRINTF("*"); + } + + if (upd) { + seed = next_rand(seed); + bool item_op = seed & 1; + + if (work->elem[elem_idx].inserted) { + if (item_op) { + cht_remove_item(work->h, &work->elem[elem_idx].link); + } else { + void *key = (void*)work->elem[elem_idx].key; + if (1 != cht_remove_key(work->h, key)) { + TPRINTF("Err: did not rm the key\n"); + work->failed = true; + } + } + work->elem[elem_idx].inserted = false; + } else if (work->elem[elem_idx].deleted) { + work->elem[elem_idx].deleted = false; + + if (item_op) { + if (!cht_insert_unique(work->h, &work->elem[elem_idx].link)) { + TPRINTF("Err: already inserted\n"); + work->failed = true; + } + } else { + cht_insert(work->h, &work->elem[elem_idx].link); + } + + work->elem[elem_idx].inserted = true; + } + } else { + rcu_read_lock(); + cht_link_t *item = + cht_find(work->h, (void*)work->elem[elem_idx].key); + rcu_read_unlock(); + + if (item) { + if (!work->elem[elem_idx].inserted) { + TPRINTF("Err: found but not inserted!"); + work->failed = true; + } + if (item != &work->elem[elem_idx].link) { + TPRINTF("Err: found but incorrect item\n"); + work->failed = true; + } + } else { + if (work->elem[elem_idx].inserted) { + TPRINTF("Err: inserted but not found!"); + work->failed = true; + } + } + } + } + + + /* Remove anything we may have inserted. */ + for (size_t i = 0; i < work->elem_cnt; ++i) { + void *key = (void*) work->elem[i].key; + cht_remove_key(work->h, key); + } +} + +static bool do_stress(void) +{ + cht_t h; + + if (!cht_create(&h, 0, 0, 0, &stress_ops)) { + TPRINTF("Failed to create the table\n"); + return false; + } + + const size_t wave_cnt = 10; + const size_t max_thread_cnt = 8; + const size_t resize_thread_cnt = 2; + size_t op_thread_cnt = min(max_thread_cnt, 2 * config.cpu_active); + size_t total_thr_cnt = op_thread_cnt + resize_thread_cnt; + size_t items_per_thread = 1024; + + size_t work_cnt = op_thread_cnt + resize_thread_cnt; + size_t item_cnt = op_thread_cnt * items_per_thread; + + /* Alloc hash table items. */ + size_t size = item_cnt * sizeof(stress_t) + work_cnt * sizeof(stress_work_t) + + sizeof(int); + + TPRINTF("Alloc and init table items. \n"); + void *p = malloc(size, FRAME_ATOMIC); + if (!p) { + TPRINTF("Failed to alloc items\n"); + cht_destroy(&h); + return false; + } + + stress_t *pitem = p + work_cnt * sizeof(stress_work_t); + stress_work_t *pwork = p; + int *pstop = (int*)(pitem + item_cnt); + + *pstop = 0; + + /* Init work items. */ + for (size_t i = 0; i < op_thread_cnt; ++i) { + pwork[i].h = &h; + pwork[i].stop = pstop; + pwork[i].elem = &pitem[i * items_per_thread]; + pwork[i].upd_prob = (i + 1) * 100 / op_thread_cnt; + pwork[i].id = i; + pwork[i].elem_cnt = items_per_thread; + pwork[i].failed = false; + } + + for (size_t i = op_thread_cnt; i < op_thread_cnt + resize_thread_cnt; ++i) { + pwork[i].h = &h; + pwork[i].stop = pstop; + pwork[i].wave_cnt = wave_cnt; + pwork[i].wave_elems = item_cnt * 4; + pwork[i].id = i; + pwork[i].failed = false; + } + + /* Init table elements. */ + for (size_t k = 0; k < op_thread_cnt; ++k) { + for (size_t i = 0; i < items_per_thread; ++i) { + pwork[k].elem[i].key = (i << 8) + k; + pwork[k].elem[i].free = false; + pwork[k].elem[i].inserted = false; + pwork[k].elem[i].deleted = true; + } + } + + TPRINTF("Running %zu ins/del/find stress threads + %zu resizers.\n", + op_thread_cnt, resize_thread_cnt); + + /* Create and run threads. */ + thread_t *thr[max_thread_cnt + resize_thread_cnt]; + + for (size_t i = 0; i < total_thr_cnt; ++i) { + if (i < op_thread_cnt) + thr[i] = thread_create(op_stresser, &pwork[i], TASK, 0, "cht-op-stress"); + else + thr[i] = thread_create(resize_stresser, &pwork[i], TASK, 0, "cht-resize"); + + ASSERT(thr[i]); + thread_wire(thr[i], &cpus[i % config.cpu_active]); + thread_ready(thr[i]); + } + + bool failed = false; + + /* Wait for all threads to return. */ + TPRINTF("Joining resize stressers.\n"); + for (size_t i = op_thread_cnt; i < total_thr_cnt; ++i) { + thread_join(thr[i]); + failed = pwork[i].failed || failed; + } + + TPRINTF("Joining op stressers.\n"); + for (int i = (int)op_thread_cnt - 1; i >= 0; --i) { + TPRINTF("%d threads remain\n", i); + thread_join(thr[i]); + failed = pwork[i].failed || failed; + } + + cht_destroy(&h); + free(p); + + return !failed; +} + +/*-------------------------------------------------------------------*/ + + +const char *test_cht1(void) +{ + const char *err = sanity_test(); + if (err) + return err; + printf("Basic sanity test: ok.\n"); + + if (!do_stress()) + return "CHT stress test failed."; + else + return 0; +} Index: kernel/test/cht/cht1.def =================================================================== --- kernel/test/cht/cht1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/cht/cht1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,6 @@ +{ + "cht", + "Concurrent hash table test", + &test_cht1, + true +}, Index: kernel/test/smpcall/smpcall1.c =================================================================== --- kernel/test/smpcall/smpcall1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/smpcall/smpcall1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,158 @@ +/* + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* + * Maximum total number of smp_calls in the system is: + * 162000 == 9^2 * 1000 * 2 + * == MAX_CPUS^2 * ITERATIONS * EACH_CPU_INC_PER_ITER + */ +#define MAX_CPUS 9 +#define ITERATIONS 1000 +#define EACH_CPU_INC_PER_ITER 2 + + +static void inc(void *p) +{ + ASSERT(interrupts_disabled()); + + size_t *pcall_cnt = (size_t*)p; + /* + * No synchronization. Tests if smp_calls makes changes + * visible to the caller. + */ + ++*pcall_cnt; +} + + +static void test_thread(void *p) +{ + size_t *pcall_cnt = (size_t*)p; + smp_call_t call_info[MAX_CPUS]; + + unsigned int cpu_count = min(config.cpu_active, MAX_CPUS); + + for (int iter = 0; iter < ITERATIONS; ++iter) { + /* Synchronous version. */ + for (unsigned cpu_id = 0; cpu_id < cpu_count; ++cpu_id) { + /* + * smp_call should make changes by inc() visible on this cpu. + * As a result we can pass it our pcall_cnt and not worry + * about other synchronization. + */ + smp_call(cpu_id, inc, pcall_cnt); + } + + /* + * Async calls run in parallel on different cpus, so passing the + * same counter would clobber it without additional synchronization. + */ + size_t local_cnt[MAX_CPUS] = {0}; + + /* Now start asynchronous calls. */ + for (unsigned cpu_id = 0; cpu_id < cpu_count; ++cpu_id) { + smp_call_async(cpu_id, inc, &local_cnt[cpu_id], &call_info[cpu_id]); + } + + /* And wait for all async calls to complete. */ + for (unsigned cpu_id = 0; cpu_id < cpu_count; ++cpu_id) { + smp_call_wait(&call_info[cpu_id]); + *pcall_cnt += local_cnt[cpu_id]; + } + + /* Give other threads a chance to run. */ + thread_usleep(10000); + } +} + +static size_t calc_exp_calls(size_t thread_cnt) +{ + return thread_cnt * ITERATIONS * EACH_CPU_INC_PER_ITER; +} + +const char *test_smpcall1(void) +{ + /* Number of received calls that were sent by cpu[i]. */ + size_t call_cnt[MAX_CPUS] = {0}; + thread_t *thread[MAX_CPUS] = {0}; + + unsigned int cpu_count = min(config.cpu_active, MAX_CPUS); + size_t running_thread_cnt = 0; + + TPRINTF("Spawning threads on %u cpus.\n", cpu_count); + + /* Create a wired thread on each cpu. */ + for (unsigned int id = 0; id < cpu_count; ++id) { + thread[id] = thread_create(test_thread, &call_cnt[id], TASK, + THREAD_FLAG_NONE, "smp-call-test"); + + if (thread[id]) { + thread_wire(thread[id], &cpus[id]); + ++running_thread_cnt; + } else { + TPRINTF("Failed to create thread on cpu%u.\n", id); + } + } + + size_t exp_calls = calc_exp_calls(running_thread_cnt); + size_t exp_calls_sum = exp_calls * cpu_count; + + TPRINTF("Running %zu wired threads. Expecting %zu calls. Be patient.\n", + running_thread_cnt, exp_calls_sum); + + for (unsigned int i = 0; i < cpu_count; ++i) { + if (thread[i] != NULL) { + thread_ready(thread[i]); + } + } + + /* Wait for threads to complete. */ + for (unsigned int i = 0; i < cpu_count; ++i) { + if (thread[i] != NULL) { + thread_join(thread[i]); + thread_detach(thread[i]); + } + } + + TPRINTF("Threads finished. Checking number of smp_call()s.\n"); + + bool ok = true; + size_t calls_sum = 0; + + for (size_t i = 0; i < cpu_count; ++i) { + if (thread[i] != NULL) { + if (call_cnt[i] != exp_calls) { + ok = false; + TPRINTF("Error: %zu instead of %zu cpu%zu's calls were" + " acknowledged.\n", call_cnt[i], exp_calls, i); + } + } + + calls_sum += call_cnt[i]; + } + + if (calls_sum != exp_calls_sum) { + TPRINTF("Error: total acknowledged sum: %zu instead of %zu.\n", + calls_sum, exp_calls_sum); + + ok = false; + } + + if (ok) { + TPRINTF("Success: number of received smp_calls is as expected (%zu).\n", + exp_calls_sum); + return NULL; + } else + return "Failed: incorrect acknowledged smp_calls.\n"; + +} Index: kernel/test/smpcall/smpcall1.def =================================================================== --- kernel/test/smpcall/smpcall1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/smpcall/smpcall1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,6 @@ +{ + "smpcall1", + "smp_call() test", + &test_smpcall1, + true +}, Index: kernel/test/synch/rcu1.c =================================================================== --- kernel/test/synch/rcu1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/rcu1.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,1052 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "abi/errno.h" +#include "time/delay.h" + +#define MAX_THREADS 32 + +static int one_idx = 0; +static thread_t *thread[MAX_THREADS] = {0}; + +typedef struct { + rcu_item_t rcu; + bool exited; +} exited_t; + +/* Callback raced with preexisting readers. */ +#define ERACE 123 +/* Waited for too long for the callback to exit; consider it lost. */ +#define ECBLOST 432 + +/*-------------------------------------------------------------------*/ +static void wait_for_cb_exit(size_t secs, exited_t *p, int *presult) +{ + size_t loops = 0; + /* 4 secs max */ + size_t loop_ms_sec = 500; + size_t max_loops = ((secs * 1000 + loop_ms_sec - 1) / loop_ms_sec); + + while (loops < max_loops && !p->exited) { + ++loops; + thread_usleep(loop_ms_sec * 1000); + TPRINTF("."); + } + + if (!p->exited) { + *presult = ECBLOST; + } +} + +static size_t get_thread_cnt(void) +{ + return min(MAX_THREADS, config.cpu_active * 4); +} + +static void run_thread(size_t k, void (*func)(void*), void *arg) +{ + ASSERT(thread[k] == NULL); + + thread[k] = thread_create(func, arg, TASK, THREAD_FLAG_NONE, + "test-rcu-thread"); + + if(thread[k]) { + /* Distribute evenly. */ + thread_wire(thread[k], &cpus[k % config.cpu_active]); + thread_ready(thread[k]); + } +} + +static void run_all(void (*func)(void*)) +{ + size_t thread_cnt = get_thread_cnt(); + + one_idx = 0; + + for (size_t i = 0; i < thread_cnt; ++i) { + run_thread(i, func, 0); + } +} + +static void join_all(void) +{ + size_t thread_cnt = get_thread_cnt(); + + one_idx = 0; + + for (size_t i = 0; i < thread_cnt; ++i) { + if (thread[i]) { + bool joined = false; + do { + int ret = thread_join_timeout(thread[i], 5 * 1000 * 1000, 0); + joined = (ret != ESYNCH_TIMEOUT); + + if (ret == ESYNCH_OK_BLOCKED) { + TPRINTF("%zu threads remain\n", thread_cnt - i - 1); + } + } while (!joined); + + thread_detach(thread[i]); + thread[i] = 0; + } + } +} + +static void run_one(void (*func)(void*), void *arg) +{ + ASSERT(one_idx < MAX_THREADS); + run_thread(one_idx, func, arg); + ++one_idx; +} + + +static void join_one(void) +{ + ASSERT(0 < one_idx && one_idx <= MAX_THREADS); + + --one_idx; + + if (thread[one_idx]) { + thread_join(thread[one_idx]); + thread_detach(thread[one_idx]); + thread[one_idx] = 0; + } +} + +/*-------------------------------------------------------------------*/ + + +static void nop_reader(void *arg) +{ + size_t nop_iters = (size_t)arg; + + TPRINTF("Enter nop-reader\n"); + + for (size_t i = 0; i < nop_iters; ++i) { + rcu_read_lock(); + rcu_read_unlock(); + } + + TPRINTF("Exit nop-reader\n"); +} + +static void get_seq(size_t from, size_t to, size_t steps, size_t *seq) +{ + ASSERT(0 < steps && from <= to && 0 < to); + size_t inc = (to - from) / (steps - 1); + + for (size_t i = 0; i < steps - 1; ++i) { + seq[i] = i * inc + from; + } + + seq[steps - 1] = to; +} + +static bool do_nop_readers(void) +{ + size_t seq[MAX_THREADS] = {0}; + get_seq(100, 100000, get_thread_cnt(), seq); + + TPRINTF("\nRun %zu thr: repeat empty no-op reader sections\n", get_thread_cnt()); + + for (size_t k = 0; k < get_thread_cnt(); ++k) + run_one(nop_reader, (void*)seq[k]); + + TPRINTF("\nJoining %zu no-op readers\n", get_thread_cnt()); + join_all(); + + return true; +} + +/*-------------------------------------------------------------------*/ + + + +static void long_reader(void *arg) +{ + const size_t iter_cnt = 100 * 1000 * 1000; + size_t nop_iters = (size_t)arg; + size_t outer_iters = iter_cnt / nop_iters; + + TPRINTF("Enter long-reader\n"); + + for (size_t i = 0; i < outer_iters; ++i) { + rcu_read_lock(); + + for (volatile size_t k = 0; k < nop_iters; ++k) { + /* nop, but increment volatile k */ + } + + rcu_read_unlock(); + } + + TPRINTF("Exit long-reader\n"); +} + +static bool do_long_readers(void) +{ + size_t seq[MAX_THREADS] = {0}; + get_seq(10, 1000 * 1000, get_thread_cnt(), seq); + + TPRINTF("\nRun %zu thr: repeat long reader sections, will preempt, no cbs.\n", + get_thread_cnt()); + + for (size_t k = 0; k < get_thread_cnt(); ++k) + run_one(long_reader, (void*)seq[k]); + + TPRINTF("\nJoining %zu readers with long reader sections.\n", get_thread_cnt()); + join_all(); + + return true; +} + +/*-------------------------------------------------------------------*/ + + +static atomic_t nop_callbacks_cnt = {0}; +/* Must be even. */ +static const int nop_updater_iters = 10000; + +static void count_cb(rcu_item_t *item) +{ + atomic_inc(&nop_callbacks_cnt); + free(item); +} + +static void nop_updater(void *arg) +{ + for (int i = 0; i < nop_updater_iters; i += 2){ + rcu_item_t *a = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); + rcu_item_t *b = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); + + if (a && b) { + rcu_call(a, count_cb); + rcu_call(b, count_cb); + } else { + TPRINTF("[out-of-mem]\n"); + free(a); + free(b); + return; + } + } +} + +static bool do_nop_callbacks(void) +{ + atomic_set(&nop_callbacks_cnt, 0); + + size_t exp_cnt = nop_updater_iters * get_thread_cnt(); + size_t max_used_mem = sizeof(rcu_item_t) * exp_cnt; + + TPRINTF("\nRun %zu thr: post %zu no-op callbacks (%zu B used), no readers.\n", + get_thread_cnt(), exp_cnt, max_used_mem); + + run_all(nop_updater); + TPRINTF("\nJoining %zu no-op callback threads\n", get_thread_cnt()); + join_all(); + + size_t loop_cnt = 0, max_loops = 15; + + while (exp_cnt != atomic_get(&nop_callbacks_cnt) && loop_cnt < max_loops) { + ++loop_cnt; + TPRINTF("."); + thread_sleep(1); + } + + return loop_cnt < max_loops; +} + +/*-------------------------------------------------------------------*/ + +typedef struct { + rcu_item_t rcu_item; + int cookie; +} item_w_cookie_t; + +const int magic_cookie = 0x01234567; +static int one_cb_is_done = 0; + +static void one_cb_done(rcu_item_t *item) +{ + ASSERT( ((item_w_cookie_t *)item)->cookie == magic_cookie); + one_cb_is_done = 1; + TPRINTF("Callback()\n"); + free(item); +} + +static void one_cb_reader(void *arg) +{ + TPRINTF("Enter one-cb-reader\n"); + + rcu_read_lock(); + + item_w_cookie_t *item = malloc(sizeof(item_w_cookie_t), FRAME_ATOMIC); + + if (item) { + item->cookie = magic_cookie; + rcu_call(&item->rcu_item, one_cb_done); + } else { + TPRINTF("\n[out-of-mem]\n"); + } + + thread_sleep(1); + + rcu_read_unlock(); + + TPRINTF("Exit one-cb-reader\n"); +} + +static bool do_one_cb(void) +{ + one_cb_is_done = 0; + + TPRINTF("\nRun a single reader that posts one callback.\n"); + run_one(one_cb_reader, 0); + join_one(); + + TPRINTF("\nJoined one-cb reader, wait for callback.\n"); + size_t loop_cnt = 0; + size_t max_loops = 4; /* 200 ms total */ + + while (!one_cb_is_done && loop_cnt < max_loops) { + thread_usleep(50 * 1000); + ++loop_cnt; + } + + return one_cb_is_done; +} + +/*-------------------------------------------------------------------*/ + +typedef struct { + size_t update_cnt; + size_t read_cnt; + size_t iters; +} seq_work_t; + +typedef struct { + rcu_item_t rcu; + atomic_count_t start_time; +} seq_item_t; + + +static int seq_test_result = EOK; + +static atomic_t cur_time = {1}; +static atomic_count_t max_upd_done_time = {0}; + +static void seq_cb(rcu_item_t *rcu_item) +{ + seq_item_t *item = member_to_inst(rcu_item, seq_item_t, rcu); + + /* Racy but errs to the conservative side, so it is ok. */ + if (max_upd_done_time < item->start_time) { + max_upd_done_time = item->start_time; + + /* Make updated time visible */ + memory_barrier(); + } + + free(item); +} + +static void seq_func(void *arg) +{ + seq_work_t *work = (seq_work_t*)arg; + + /* Alternate between reader and updater roles. */ + for (size_t k = 0; k < work->iters; ++k) { + /* Reader */ + for (size_t i = 0; i < work->read_cnt; ++i) { + rcu_read_lock(); + atomic_count_t start_time = atomic_postinc(&cur_time); + + for (volatile size_t d = 0; d < 10 * i; ++d ){ + /* no-op */ + } + + /* Get most recent max_upd_done_time. */ + memory_barrier(); + + if (start_time < max_upd_done_time) { + seq_test_result = ERACE; + } + + rcu_read_unlock(); + + if (seq_test_result != EOK) + return; + } + + /* Updater */ + for (size_t i = 0; i < work->update_cnt; ++i) { + seq_item_t *a = malloc(sizeof(seq_item_t), FRAME_ATOMIC); + seq_item_t *b = malloc(sizeof(seq_item_t), FRAME_ATOMIC); + + if (a && b) { + a->start_time = atomic_postinc(&cur_time); + rcu_call(&a->rcu, seq_cb); + + b->start_time = atomic_postinc(&cur_time); + rcu_call(&b->rcu, seq_cb); + } else { + TPRINTF("\n[out-of-mem]\n"); + seq_test_result = ENOMEM; + free(a); + free(b); + return; + } + } + + } +} + +static bool do_seq_check(void) +{ + seq_test_result = EOK; + max_upd_done_time = 0; + atomic_set(&cur_time, 1); + + const size_t iters = 100; + const size_t total_cnt = 1000; + size_t read_cnt[MAX_THREADS] = {0}; + seq_work_t item[MAX_THREADS]; + + size_t total_cbs = 0; + size_t max_used_mem = 0; + + get_seq(0, total_cnt, get_thread_cnt(), read_cnt); + + + for (size_t i = 0; i < get_thread_cnt(); ++i) { + item[i].update_cnt = total_cnt - read_cnt[i]; + item[i].read_cnt = read_cnt[i]; + item[i].iters = iters; + + total_cbs += 2 * iters * item[i].update_cnt; + } + + max_used_mem = total_cbs * sizeof(seq_item_t); + + const char *mem_suffix; + uint64_t mem_units; + bin_order_suffix(max_used_mem, &mem_units, &mem_suffix, false); + + TPRINTF("\nRun %zu th: check callback completion time in readers. " + "%zu callbacks total (max %" PRIu64 " %s used). Be patient.\n", + get_thread_cnt(), total_cbs, mem_units, mem_suffix); + + for (size_t i = 0; i < get_thread_cnt(); ++i) { + run_one(seq_func, &item[i]); + } + + TPRINTF("\nJoining %zu seq-threads\n", get_thread_cnt()); + join_all(); + + if (seq_test_result == ENOMEM) { + TPRINTF("\nErr: out-of mem\n"); + } else if (seq_test_result == ERACE) { + TPRINTF("\nERROR: race detected!!\n"); + } + + return seq_test_result == EOK; +} + +/*-------------------------------------------------------------------*/ + + +static void reader_unlocked(rcu_item_t *item) +{ + exited_t *p = (exited_t*)item; + p->exited = true; +} + +static void reader_exit(void *arg) +{ + rcu_read_lock(); + rcu_read_lock(); + rcu_read_lock(); + rcu_read_unlock(); + + rcu_call((rcu_item_t*)arg, reader_unlocked); + + rcu_read_lock(); + rcu_read_lock(); + + /* Exit without unlocking the rcu reader section. */ +} + +static bool do_reader_exit(void) +{ + TPRINTF("\nReader exits thread with rcu_lock\n"); + + exited_t *p = malloc(sizeof(exited_t), FRAME_ATOMIC); + if (!p) { + TPRINTF("[out-of-mem]\n"); + return false; + } + + p->exited = false; + + run_one(reader_exit, p); + join_one(); + + int result = EOK; + wait_for_cb_exit(2 /* secs */, p, &result); + + if (result != EOK) { + TPRINTF("Err: RCU locked up after exiting from within a reader\n"); + /* Leak the mem. */ + } else { + free(p); + } + + return result == EOK; +} + +/*-------------------------------------------------------------------*/ + +/*-------------------------------------------------------------------*/ + +typedef struct preempt_struct { + exited_t e; + int result; +} preempt_t; + + +static void preempted_unlocked(rcu_item_t *item) +{ + preempt_t *p = member_to_inst(item, preempt_t, e.rcu); + p->e.exited = true; + TPRINTF("Callback().\n"); +} + +static void preempted_reader_prev(void *arg) +{ + preempt_t *p = (preempt_t*)arg; + ASSERT(!p->e.exited); + + TPRINTF("reader_prev{ "); + + rcu_read_lock(); + scheduler(); + rcu_read_unlock(); + + /* + * Start GP after exiting reader section w/ preemption. + * Just check that the callback does not lock up and is not lost. + */ + rcu_call(&p->e.rcu, preempted_unlocked); + + TPRINTF("}reader_prev\n"); +} + +static void preempted_reader_inside_cur(void *arg) +{ + preempt_t *p = (preempt_t*)arg; + ASSERT(!p->e.exited); + + TPRINTF("reader_inside_cur{ "); + /* + * Start a GP and try to finish the reader before + * the GP ends (including preemption). + */ + rcu_call(&p->e.rcu, preempted_unlocked); + + /* Give RCU threads a chance to start up. */ + scheduler(); + scheduler(); + + rcu_read_lock(); + /* Come back as soon as possible to complete before GP ends. */ + thread_usleep(2); + rcu_read_unlock(); + + TPRINTF("}reader_inside_cur\n"); +} + + +static void preempted_reader_cur(void *arg) +{ + preempt_t *p = (preempt_t*)arg; + ASSERT(!p->e.exited); + + TPRINTF("reader_cur{ "); + rcu_read_lock(); + + /* Start GP. */ + rcu_call(&p->e.rcu, preempted_unlocked); + + /* Preempt while cur GP detection is running */ + thread_sleep(1); + + /* Err: exited before this reader completed. */ + if (p->e.exited) + p->result = ERACE; + + rcu_read_unlock(); + TPRINTF("}reader_cur\n"); +} + +static void preempted_reader_next1(void *arg) +{ + preempt_t *p = (preempt_t*)arg; + ASSERT(!p->e.exited); + + TPRINTF("reader_next1{ "); + rcu_read_lock(); + + /* Preempt before cur GP detection starts. */ + scheduler(); + + /* Start GP. */ + rcu_call(&p->e.rcu, preempted_unlocked); + + /* Err: exited before this reader completed. */ + if (p->e.exited) + p->result = ERACE; + + rcu_read_unlock(); + TPRINTF("}reader_next1\n"); +} + +static void preempted_reader_next2(void *arg) +{ + preempt_t *p = (preempt_t*)arg; + ASSERT(!p->e.exited); + + TPRINTF("reader_next2{ "); + rcu_read_lock(); + + /* Preempt before cur GP detection starts. */ + scheduler(); + + /* Start GP. */ + rcu_call(&p->e.rcu, preempted_unlocked); + + /* + * Preempt twice while GP is running after we've been known + * to hold up the GP just to make sure multiple preemptions + * are properly tracked if a reader is delaying the cur GP. + */ + thread_sleep(1); + thread_sleep(1); + + /* Err: exited before this reader completed. */ + if (p->e.exited) + p->result = ERACE; + + rcu_read_unlock(); + TPRINTF("}reader_next2\n"); +} + + +static bool do_one_reader_preempt(void (*f)(void*), const char *err) +{ + preempt_t *p = malloc(sizeof(preempt_t), FRAME_ATOMIC); + if (!p) { + TPRINTF("[out-of-mem]\n"); + return false; + } + + p->e.exited = false; + p->result = EOK; + + run_one(f, p); + join_one(); + + /* Wait at most 4 secs. */ + wait_for_cb_exit(4, &p->e, &p->result); + + if (p->result == EOK) { + free(p); + return true; + } else { + TPRINTF(err); + /* Leak a bit of mem. */ + return false; + } +} + +static bool do_reader_preempt(void) +{ + TPRINTF("\nReaders will be preempted.\n"); + + bool success = true; + bool ok = true; + + ok = do_one_reader_preempt(preempted_reader_prev, + "Err: preempted_reader_prev()\n"); + success = success && ok; + + ok = do_one_reader_preempt(preempted_reader_inside_cur, + "Err: preempted_reader_inside_cur()\n"); + success = success && ok; + + ok = do_one_reader_preempt(preempted_reader_cur, + "Err: preempted_reader_cur()\n"); + success = success && ok; + + ok = do_one_reader_preempt(preempted_reader_next1, + "Err: preempted_reader_next1()\n"); + success = success && ok; + + ok = do_one_reader_preempt(preempted_reader_next2, + "Err: preempted_reader_next2()\n"); + success = success && ok; + + return success; +} + +/*-------------------------------------------------------------------*/ +typedef struct { + bool reader_done; + bool reader_running; + bool synch_running; +} synch_t; + +static void synch_reader(void *arg) +{ + synch_t *synch = (synch_t *) arg; + + rcu_read_lock(); + + /* Order accesses of synch after the reader section begins. */ + memory_barrier(); + + synch->reader_running = true; + + while (!synch->synch_running) { + /* 0.5 sec */ + delay(500 * 1000); + } + + /* Run for 1 sec */ + delay(1000 * 1000); + /* thread_join() propagates done to do_synch() */ + synch->reader_done = true; + + rcu_read_unlock(); +} + + +static bool do_synch(void) +{ + TPRINTF("\nSynchronize with long reader\n"); + + synch_t *synch = malloc(sizeof(synch_t), FRAME_ATOMIC); + + if (!synch) { + TPRINTF("[out-of-mem]\n"); + return false; + } + + synch->reader_done = false; + synch->reader_running = false; + synch->synch_running = false; + + run_one(synch_reader, synch); + + /* Wait for the reader to enter its critical section. */ + scheduler(); + while (!synch->reader_running) { + thread_usleep(500 * 1000); + } + + synch->synch_running = true; + + rcu_synchronize(); + join_one(); + + + if (synch->reader_done) { + free(synch); + return true; + } else { + TPRINTF("Err: synchronize() exited prematurely \n"); + /* Leak some mem. */ + return false; + } +} + +/*-------------------------------------------------------------------*/ +typedef struct { + rcu_item_t rcu_item; + atomic_t done; +} barrier_t; + +static void barrier_callback(rcu_item_t *item) +{ + barrier_t *b = member_to_inst(item, barrier_t, rcu_item); + atomic_set(&b->done, 1); +} + +static bool do_barrier(void) +{ + TPRINTF("\nrcu_barrier: Wait for outstanding rcu callbacks to complete\n"); + + barrier_t *barrier = malloc(sizeof(barrier_t), FRAME_ATOMIC); + + if (!barrier) { + TPRINTF("[out-of-mem]\n"); + return false; + } + + atomic_set(&barrier->done, 0); + + rcu_call(&barrier->rcu_item, barrier_callback); + rcu_barrier(); + + if (1 == atomic_get(&barrier->done)) { + free(barrier); + return true; + } else { + TPRINTF("rcu_barrier() exited prematurely.\n"); + /* Leak some mem. */ + return false; + } +} + +/*-------------------------------------------------------------------*/ + +typedef struct { + size_t iters; + bool master; +} stress_t; + + +static void stress_reader(void *arg) +{ + bool *done = (bool*) arg; + + while (!*done) { + rcu_read_lock(); + rcu_read_unlock(); + + /* + * Do some work outside of the reader section so we are not always + * preempted in the reader section. + */ + delay(5); + } +} + +static void stress_cb(rcu_item_t *item) +{ + /* 5 us * 1000 * 1000 iters == 5 sec per updater thread */ + delay(5); + free(item); +} + +static void stress_updater(void *arg) +{ + stress_t *s = (stress_t *)arg; + + for (size_t i = 0; i < s->iters; ++i) { + rcu_item_t *item = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); + + if (item) { + rcu_call(item, stress_cb); + } else { + TPRINTF("[out-of-mem]\n"); + return; + } + + /* Print a dot if we make a progress of 1% */ + if (s->master && 0 == (i % (s->iters/100))) + TPRINTF("."); + } +} + +static bool do_stress(void) +{ + size_t cb_per_thread = 1000 * 1000; + bool done = false; + stress_t master = { .iters = cb_per_thread, .master = true }; + stress_t worker = { .iters = cb_per_thread, .master = false }; + + size_t thread_cnt = min(MAX_THREADS / 2, config.cpu_active); + /* Each cpu has one reader and one updater. */ + size_t reader_cnt = thread_cnt; + size_t updater_cnt = thread_cnt; + + size_t exp_upd_calls = updater_cnt * cb_per_thread; + size_t max_used_mem = exp_upd_calls * sizeof(rcu_item_t); + + const char *mem_suffix; + uint64_t mem_units; + bin_order_suffix(max_used_mem, &mem_units, &mem_suffix, false); + + TPRINTF("\nStress: Run %zu nop-readers and %zu updaters. %zu callbacks" + " total (max %" PRIu64 " %s used). Be very patient.\n", + reader_cnt, updater_cnt, exp_upd_calls, mem_units, mem_suffix); + + for (size_t k = 0; k < reader_cnt; ++k) { + run_one(stress_reader, &done); + } + + for (size_t k = 0; k < updater_cnt; ++k) { + run_one(stress_updater, k > 0 ? &worker : &master); + } + + TPRINTF("\nJoining %zu stress updaters.\n", updater_cnt); + + for (size_t k = 0; k < updater_cnt; ++k) { + join_one(); + } + + done = true; + + TPRINTF("\nJoining %zu stress nop-readers.\n", reader_cnt); + + join_all(); + return true; +} +/*-------------------------------------------------------------------*/ + +typedef struct { + rcu_item_t r; + size_t total_cnt; + size_t count_down; + bool expedite; +} expedite_t; + +static void expedite_cb(rcu_item_t *arg) +{ + expedite_t *e = (expedite_t *)arg; + + if (1 < e->count_down) { + --e->count_down; + + if (0 == (e->count_down % (e->total_cnt/100))) { + TPRINTF("*"); + } + + _rcu_call(e->expedite, &e->r, expedite_cb); + } else { + /* Do not touch any of e's mem after we declare we're done with it. */ + memory_barrier(); + e->count_down = 0; + } +} + +static void run_expedite(bool exp, size_t cnt) +{ + expedite_t e; + e.total_cnt = cnt; + e.count_down = cnt; + e.expedite = exp; + + _rcu_call(e.expedite, &e.r, expedite_cb); + + while (0 < e.count_down) { + thread_sleep(1); + TPRINTF("."); + } +} + +static bool do_expedite(void) +{ + size_t exp_cnt = 1000 * 1000; + size_t normal_cnt = 1 * 1000; + + TPRINTF("Expedited: sequence of %zu rcu_calls\n", exp_cnt); + run_expedite(true, exp_cnt); + TPRINTF("Normal/non-expedited: sequence of %zu rcu_calls\n", normal_cnt); + run_expedite(false, normal_cnt); + return true; +} +/*-------------------------------------------------------------------*/ + +struct test_func { + bool include; + bool (*func)(void); + const char *desc; +}; + + +const char *test_rcu1(void) +{ + struct test_func test_func[] = { + { 1, do_one_cb, "do_one_cb" }, + { 1, do_reader_preempt, "do_reader_preempt" }, + { 1, do_synch, "do_synch" }, + { 1, do_barrier, "do_barrier" }, + { 1, do_reader_exit, "do_reader_exit" }, + { 1, do_nop_readers, "do_nop_readers" }, + { 1, do_seq_check, "do_seq_check" }, + { 0, do_long_readers, "do_long_readers" }, + { 1, do_nop_callbacks, "do_nop_callbacks" }, + { 0, do_expedite, "do_expedite" }, + { 1, do_stress, "do_stress" }, + { 0, 0, 0 } + }; + + bool success = true; + bool ok = true; + uint64_t completed_gps = rcu_completed_gps(); + uint64_t delta_gps = 0; + + for (int i = 0; test_func[i].func != 0; ++i) { + if (!test_func[i].include) { + TPRINTF("\nSubtest %s() skipped.\n", test_func[i].desc); + continue; + } else { + TPRINTF("\nRunning subtest %s.\n", test_func[i].desc); + } + + ok = test_func[i].func(); + success = success && ok; + + delta_gps = rcu_completed_gps() - completed_gps; + completed_gps += delta_gps; + + if (ok) { + TPRINTF("\nSubtest %s() ok (GPs: %" PRIu64 ").\n", + test_func[i].desc, delta_gps); + } else { + TPRINTF("\nFailed: %s(). Pausing for 5 secs.\n", test_func[i].desc); + thread_sleep(5); + } + } + + if (success) + return 0; + else + return "One of the tests failed."; +} Index: kernel/test/synch/rcu1.def =================================================================== --- kernel/test/synch/rcu1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/rcu1.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,6 @@ +{ + "rcu1", + "Basic RCU test", + &test_rcu1, + true +}, Index: kernel/test/synch/workq-test-core.h =================================================================== --- kernel/test/synch/workq-test-core.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/workq-test-core.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + + +typedef struct test_work { + work_t work_item; + int master; + int wave; + int count_down; +} test_work_t; + +static atomic_t call_cnt[WAVES]; + + +/* Fwd decl - implement in your actual test file.. */ +static int core_workq_enqueue(work_t *work_item, work_func_t func); + + +static bool new_wave(test_work_t *work) +{ + ++work->wave; + + if (work->wave < WAVES) { + work->count_down = COUNT; + return true; + } else { + return false; + } +} + + +static int is_pow2(int num) +{ + unsigned n = (unsigned)num; + return (n != 0) && 0 == (n & (n-1)); +} + +static test_work_t * create_child(test_work_t *work) +{ + test_work_t *child = malloc(sizeof(test_work_t), 0); + ASSERT(child); + if (child) { + child->master = false; + child->wave = work->wave; + child->count_down = work->count_down; + } + + return child; +} + +static void free_work(test_work_t *work) +{ + memsetb(work, sizeof(test_work_t), 0xfa); + free(work); +} + +static void reproduce(work_t *work_item) +{ + /* Ensure work_item is ours for the taking. */ + memsetb(work_item, sizeof(work_t), 0xec); + + test_work_t *work = (test_work_t *)work_item; + + atomic_inc(&call_cnt[work->wave]); + + if (0 < work->count_down) { + /* Sleep right before creating the last generation. */ + if (1 == work->count_down) { + bool sleeping_wave = ((work->wave % 2) == 1); + + /* Master never sleeps. */ + if (sleeping_wave && !work->master) { + thread_usleep(WAVE_SLEEP_MS * 1000); + } + } + + --work->count_down; + + /* + * Enqueue a child if count_down is power-of-2. + * Leads to exponential growth. + */ + if (is_pow2(work->count_down + 1)) { + test_work_t *child = create_child(work); + if (child) { + if (!core_workq_enqueue(&child->work_item, reproduce)) + free_work(child); + } + } + + if (!core_workq_enqueue(work_item, reproduce)) { + if (work->master) + TPRINTF("\nErr: Master work item exiting prematurely!\n"); + + free_work(work); + } + } else { + /* We're done with this wave - only the master survives. */ + + if (work->master && new_wave(work)) { + if (!core_workq_enqueue(work_item, reproduce)) { + TPRINTF("\nErr: Master work could not start a new wave!\n"); + free_work(work); + } + } else { + if (work->master) + TPRINTF("\nMaster work item done.\n"); + + free_work(work); + } + } +} + +static const char *run_workq_core(bool end_prematurely) +{ + for (int i = 0; i < WAVES; ++i) { + atomic_set(&call_cnt[i], 0); + } + + test_work_t *work = malloc(sizeof(test_work_t), 0); + + work->master = true; + work->wave = 0; + work->count_down = COUNT; + + /* + * k == COUNT_POW + * 2^k == COUNT + 1 + * + * We have "k" branching points. Therefore: + * exp_call_cnt == k*2^(k-1) + 2^k == (k + 2) * 2^(k-1) + */ + size_t exp_call_cnt = (COUNT_POW + 2) * (1 << (COUNT_POW - 1)); + + TPRINTF("waves: %d, count_down: %d, total expected calls: %zu\n", + WAVES, COUNT, exp_call_cnt * WAVES); + + + core_workq_enqueue(&work->work_item, reproduce); + + size_t sleep_cnt = 0; + /* At least 40 seconds total (or 2 sec to end while there's work). */ + size_t max_sleep_secs = end_prematurely ? 2 : MAIN_MAX_SLEEP_SEC; + size_t max_sleep_cnt = (max_sleep_secs * 1000) / MAIN_POLL_SLEEP_MS; + + for (int i = 0; i < WAVES; ++i) { + while (atomic_get(&call_cnt[i]) < exp_call_cnt + && sleep_cnt < max_sleep_cnt) { + TPRINTF("."); + thread_usleep(MAIN_POLL_SLEEP_MS * 1000); + ++sleep_cnt; + } + } + + bool success = true; + + for (int i = 0; i < WAVES; ++i) { + if (atomic_get(&call_cnt[i]) == exp_call_cnt) { + TPRINTF("Ok: %" PRIua " calls in wave %d, as expected.\n", + atomic_get(&call_cnt[i]), i); + } else { + success = false; + TPRINTF("Error: %" PRIua " calls in wave %d, but %zu expected.\n", + atomic_get(&call_cnt[i]), i, exp_call_cnt); + } + } + + + if (success) + return NULL; + else { + return "Failed to invoke the expected number of calls.\n"; + } +} Index: kernel/test/synch/workqueue2.c =================================================================== --- kernel/test/synch/workqueue2.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/workqueue2.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + + +#define WAVES 10 +#define COUNT_POW 12 +#define COUNT ((1 << COUNT_POW) - 1) +#define WAVE_SLEEP_MS 100 +#define MAIN_POLL_SLEEP_MS 100 +#define MAIN_MAX_SLEEP_SEC 40 + +/* + * Include the test implementation. + */ +#include "workq-test-core.h" + + +/*-------------------------------------------------------------------*/ + +static work_t basic_work; +static int basic_done = 0; + +static void basic_test_work(work_t *work_item) +{ + basic_done = 1; + TPRINTF("basic_test_work()"); +} + + +static void basic_test(void) +{ + TPRINTF("Issue a single work item.\n"); + basic_done = 0; + workq_global_enqueue(&basic_work, basic_test_work); + + while (!basic_done) { + TPRINTF("."); + thread_sleep(1); + } + + TPRINTF("\nBasic test done\n"); +} + +/*-------------------------------------------------------------------*/ + + +struct work_queue *workq = NULL; + +static int core_workq_enqueue(work_t *work_item, work_func_t func) +{ + return workq_enqueue(workq, work_item, func); +} +/*-------------------------------------------------------------------*/ + + +static const char *test_custom_workq_impl(bool stop, const char *qname) +{ + workq = workq_create(qname); + + if (!workq) { + return "Failed to create a work queue.\n"; + } + + const char *ret = run_workq_core(stop); + + TPRINTF("Stopping work queue...\n"); + workq_stop(workq); + + TPRINTF("Destroying work queue...\n"); + workq_destroy(workq); + return ret; +} + +static const char *test_custom_workq(void) +{ + TPRINTF("Stress testing a custom queue.\n"); + return test_custom_workq_impl(false, "test-workq"); +} + + +static const char *test_custom_workq_stop(void) +{ + TPRINTF("Stress testing a custom queue. Stops prematurely. " + "Errors are expected.\n"); + test_custom_workq_impl(true, "test-workq-stop"); + /* Errors are expected. */ + return 0; +} + + +const char *test_workqueue_all(void) +{ + const char *err = 0; + const char *res; + + basic_test(); + + res = test_custom_workq(); + if (res) { + TPRINTF(res); + err = res; + } + + res = test_custom_workq_stop(); + if (res) { + TPRINTF(res); + err = res; + } + + res = test_workqueue3(); + if (res) { + TPRINTF(res); + err = res; + } + + return err; +} Index: kernel/test/synch/workqueue2.def =================================================================== --- kernel/test/synch/workqueue2.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/workqueue2.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,6 @@ +{ + "workqueue", + "Separate and system work queue stress test", + &test_workqueue_all, + true +}, Index: kernel/test/synch/workqueue3.c =================================================================== --- kernel/test/synch/workqueue3.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/workqueue3.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + + +#define WAVES 10 +#define COUNT_POW 12 +#define COUNT ((1 << COUNT_POW) - 1) +#define WAVE_SLEEP_MS 100 +#define MAIN_POLL_SLEEP_MS 100 +#define MAIN_MAX_SLEEP_SEC 40 + +/* + * Include the test implementation. + */ +#include "workq-test-core.h" + + +static int core_workq_enqueue(work_t *work_item, work_func_t func) +{ + return workq_global_enqueue(work_item, func); +} + + + +static const char *do_test(bool exit_early) +{ + const char *err = 0; + TPRINTF("Stress testing system queue.\n"); + TPRINTF("First run:\n"); + err = run_workq_core(exit_early); + + if (!err) { + TPRINTF("\nSecond run:\n"); + err = run_workq_core(exit_early); + } + + TPRINTF("Done.\n"); + + return err; +} + +const char *test_workqueue3(void) +{ + return do_test(false); +} + +const char *test_workqueue3quit(void) +{ + return do_test(true); +} Index: kernel/test/synch/workqueue3.def =================================================================== --- kernel/test/synch/workqueue3.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ kernel/test/synch/workqueue3.def (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,6 @@ +{ + "workqueue3quit", + "Global work queue test, exits early", + &test_workqueue3quit, + true +}, Index: kernel/test/test.c =================================================================== --- kernel/test/test.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/test/test.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include #include #include @@ -50,4 +51,7 @@ #include #include +#include +#include +#include #include #include @@ -56,4 +60,5 @@ #include #include +#include { .name = NULL, Index: kernel/test/test.h =================================================================== --- kernel/test/test.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ kernel/test/test.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -59,4 +59,5 @@ extern const char *test_avltree1(void); extern const char *test_btree1(void); +extern const char *test_cht1(void); extern const char *test_mips1(void); extern const char *test_fault1(void); @@ -75,4 +76,10 @@ extern const char *test_print5(void); extern const char *test_thread1(void); +extern const char *test_smpcall1(void); +extern const char *test_workqueue_all(void); +extern const char *test_workqueue3(void); +extern const char *test_workqueue3quit(void); +extern const char *test_rcu1(void); + extern test_t tests[]; Index: uspace/app/mkexfat/mkexfat.c =================================================================== --- uspace/app/mkexfat/mkexfat.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/app/mkexfat/mkexfat.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -49,4 +49,5 @@ #include #include +#include #include "exfat.h" #include "upcase.h" @@ -87,5 +88,4 @@ #define FIRST_FREE_CLUSTER 2 -#define min(x, y) ((x) < (y) ? (x) : (y)) typedef struct exfat_cfg { Index: uspace/app/trace/ipcp.c =================================================================== --- uspace/app/trace/ipcp.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/app/trace/ipcp.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -38,4 +38,5 @@ #include #include +#include #include "ipc_desc.h" #include "proto.h" @@ -52,5 +53,5 @@ ipc_callid_t call_hash; - link_t link; + ht_link_t link; } pending_call_t; @@ -64,6 +65,5 @@ int have_conn[MAX_PHONE]; -#define PCALL_TABLE_CHAINS 32 -hash_table_t pending_calls; +static hash_table_t pending_calls; /* @@ -73,38 +73,32 @@ proto_t *proto_unknown; /**< Protocol with no known methods. */ -static hash_index_t pending_call_hash(unsigned long key[]); -static int pending_call_compare(unsigned long key[], hash_count_t keys, - link_t *item); -static void pending_call_remove_callback(link_t *item); - -hash_table_operations_t pending_call_ops = { + +static size_t pending_call_key_hash(void *key) +{ + ipc_callid_t *call_id = (ipc_callid_t *)key; + return *call_id; +} + +static size_t pending_call_hash(const ht_link_t *item) +{ + pending_call_t *hs = hash_table_get_inst(item, pending_call_t, link); + return hs->call_hash; +} + +static bool pending_call_key_equal(void *key, const ht_link_t *item) +{ + ipc_callid_t *call_id = (ipc_callid_t *)key; + pending_call_t *hs = hash_table_get_inst(item, pending_call_t, link); + + return *call_id == hs->call_hash; +} + +static hash_table_ops_t pending_call_ops = { .hash = pending_call_hash, - .compare = pending_call_compare, - .remove_callback = pending_call_remove_callback + .key_hash = pending_call_key_hash, + .key_equal = pending_call_key_equal, + .equal = 0, + .remove_callback = 0 }; - - -static hash_index_t pending_call_hash(unsigned long key[]) -{ -// printf("pending_call_hash\n"); - return key[0] % PCALL_TABLE_CHAINS; -} - -static int pending_call_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - pending_call_t *hs; - -// printf("pending_call_compare\n"); - hs = hash_table_get_instance(item, pending_call_t, link); - - // FIXME: this will fail if sizeof(long) < sizeof(void *). - return key[0] == hs->call_hash; -} - -static void pending_call_remove_callback(link_t *item) -{ -// printf("pending_call_remove_callback\n"); -} @@ -177,5 +171,6 @@ } - hash_table_create(&pending_calls, PCALL_TABLE_CHAINS, 1, &pending_call_ops); + bool ok = hash_table_create(&pending_calls, 0, 0, &pending_call_ops); + assert(ok); } @@ -190,5 +185,4 @@ pending_call_t *pcall; proto_t *proto; - unsigned long key[1]; oper_t *oper; sysarg_t *args; @@ -254,7 +248,5 @@ pcall->oper = oper; - key[0] = hash; - - hash_table_insert(&pending_calls, key, &pcall->link); + hash_table_insert(&pending_calls, &pcall->link); } @@ -334,7 +326,6 @@ void ipcp_call_in(ipc_call_t *call, ipc_callid_t hash) { - link_t *item; + ht_link_t *item; pending_call_t *pcall; - unsigned long key[1]; if ((hash & IPC_CALLID_ANSWERED) == 0 && hash != IPCP_CALLID_SYNC) { @@ -347,7 +338,6 @@ hash = hash & ~IPC_CALLID_ANSWERED; - key[0] = hash; - - item = hash_table_find(&pending_calls, key); + + item = hash_table_find(&pending_calls, &hash); if (item == NULL) return; /* No matching question found */ @@ -357,6 +347,6 @@ */ - pcall = hash_table_get_instance(item, pending_call_t, link); - hash_table_remove(&pending_calls, key, 1); + pcall = hash_table_get_inst(item, pending_call_t, link); + hash_table_remove(&pending_calls, &hash); parse_answer(hash, pcall, call); Index: uspace/app/trace/proto.c =================================================================== --- uspace/app/trace/proto.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/app/trace/proto.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -40,117 +40,103 @@ #include "proto.h" -#define SRV_PROTO_TABLE_CHAINS 32 -#define METHOD_OPER_TABLE_CHAINS 32 - -hash_table_t srv_proto; + +/* Maps service number to protocol */ +static hash_table_t srv_proto; typedef struct { - unsigned srv; + int srv; proto_t *proto; - link_t link; + ht_link_t link; } srv_proto_t; typedef struct { - sysarg_t method; + int method; oper_t *oper; - link_t link; + ht_link_t link; } method_oper_t; -static hash_index_t srv_proto_hash(unsigned long key[]); -static int srv_proto_compare(unsigned long key[], hash_count_t keys, - link_t *item); -static void srv_proto_remove_callback(link_t *item); - -hash_table_operations_t srv_proto_ops = { +/* Hash table operations. */ + +static size_t srv_proto_key_hash(void *key) +{ + return *(int *)key; +} + +static size_t srv_proto_hash(const ht_link_t *item) +{ + srv_proto_t *sp = hash_table_get_inst(item, srv_proto_t, link); + return sp->srv; +} + +static bool srv_proto_key_equal(void *key, const ht_link_t *item) +{ + srv_proto_t *sp = hash_table_get_inst(item, srv_proto_t, link); + return sp->srv == *(int *)key; +} + +static hash_table_ops_t srv_proto_ops = { .hash = srv_proto_hash, - .compare = srv_proto_compare, - .remove_callback = srv_proto_remove_callback + .key_hash = srv_proto_key_hash, + .key_equal = srv_proto_key_equal, + .equal = 0, + .remove_callback = 0 }; -static hash_index_t method_oper_hash(unsigned long key[]); -static int method_oper_compare(unsigned long key[], hash_count_t keys, - link_t *item); -static void method_oper_remove_callback(link_t *item); - -hash_table_operations_t method_oper_ops = { + +static size_t method_oper_key_hash(void *key) +{ + return *(int *)key; +} + +static size_t method_oper_hash(const ht_link_t *item) +{ + method_oper_t *mo = hash_table_get_inst(item, method_oper_t, link); + return mo->method; +} + +static bool method_oper_key_equal(void *key, const ht_link_t *item) +{ + method_oper_t *mo = hash_table_get_inst(item, method_oper_t, link); + return mo->method == *(int *)key; +} + +static hash_table_ops_t method_oper_ops = { .hash = method_oper_hash, - .compare = method_oper_compare, - .remove_callback = method_oper_remove_callback + .key_hash = method_oper_key_hash, + .key_equal = method_oper_key_equal, + .equal = 0, + .remove_callback = 0 }; -static hash_index_t srv_proto_hash(unsigned long key[]) -{ - return key[0] % SRV_PROTO_TABLE_CHAINS; -} - -static int srv_proto_compare(unsigned long key[], hash_count_t keys, - link_t *item) + +void proto_init(void) +{ + /* todo: check return value. */ + bool ok = hash_table_create(&srv_proto, 0, 0, &srv_proto_ops); + assert(ok); +} + +void proto_cleanup(void) +{ + hash_table_destroy(&srv_proto); +} + +void proto_register(int srv, proto_t *proto) { srv_proto_t *sp; - - sp = hash_table_get_instance(item, srv_proto_t, link); - - return key[0] == sp->srv; -} - -static void srv_proto_remove_callback(link_t *item) -{ -} - -static hash_index_t method_oper_hash(unsigned long key[]) -{ - return key[0] % METHOD_OPER_TABLE_CHAINS; -} - -static int method_oper_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - method_oper_t *mo; - - mo = hash_table_get_instance(item, method_oper_t, link); - - return key[0] == mo->method; -} - -static void method_oper_remove_callback(link_t *item) -{ -} - - -void proto_init(void) -{ - hash_table_create(&srv_proto, SRV_PROTO_TABLE_CHAINS, 1, - &srv_proto_ops); -} - -void proto_cleanup(void) -{ - hash_table_destroy(&srv_proto); -} - -void proto_register(int srv, proto_t *proto) -{ - srv_proto_t *sp; - unsigned long key; sp = malloc(sizeof(srv_proto_t)); sp->srv = srv; sp->proto = proto; - key = srv; - - hash_table_insert(&srv_proto, &key, &sp->link); + + hash_table_insert(&srv_proto, &sp->link); } proto_t *proto_get_by_srv(int srv) { - unsigned long key; - link_t *item; - srv_proto_t *sp; - - key = srv; - item = hash_table_find(&srv_proto, &key); + ht_link_t *item = hash_table_find(&srv_proto, &srv); if (item == NULL) return NULL; - sp = hash_table_get_instance(item, srv_proto_t, link); + srv_proto_t *sp = hash_table_get_inst(item, srv_proto_t, link); return sp->proto; } @@ -159,6 +145,7 @@ { proto->name = name; - hash_table_create(&proto->method_oper, SRV_PROTO_TABLE_CHAINS, 1, - &method_oper_ops); + /* todo: check return value. */ + bool ok = hash_table_create(&proto->method_oper, 0, 0, &method_oper_ops); + assert(ok); } @@ -181,25 +168,18 @@ { method_oper_t *mo; - unsigned long key; mo = malloc(sizeof(method_oper_t)); mo->method = method; mo->oper = oper; - key = method; - - hash_table_insert(&proto->method_oper, &key, &mo->link); + + hash_table_insert(&proto->method_oper, &mo->link); } oper_t *proto_get_oper(proto_t *proto, int method) { - unsigned long key; - link_t *item; - method_oper_t *mo; - - key = method; - item = hash_table_find(&proto->method_oper, &key); + ht_link_t *item = hash_table_find(&proto->method_oper, &method); if (item == NULL) return NULL; - mo = hash_table_get_instance(item, method_oper_t, link); + method_oper_t *mo = hash_table_get_inst(item, method_oper_t, link); return mo->oper; } Index: uspace/app/trace/proto.h =================================================================== --- uspace/app/trace/proto.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/app/trace/proto.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -62,6 +62,4 @@ } proto_t; -/* Maps service number to protocol */ -extern hash_table_t srv_proto; extern void proto_init(void); Index: uspace/lib/block/libblock.c =================================================================== --- uspace/lib/block/libblock.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/block/libblock.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -62,6 +62,4 @@ static LIST_INITIALIZE(dcl); -#define CACHE_BUCKETS_LOG2 10 -#define CACHE_BUCKETS (1 << CACHE_BUCKETS_LOG2) typedef struct { @@ -256,23 +254,30 @@ } -static hash_index_t cache_hash(unsigned long *key) -{ - return MERGE_LOUP32(key[0], key[1]) & (CACHE_BUCKETS - 1); -} - -static int cache_compare(unsigned long *key, hash_count_t keys, link_t *item) -{ - block_t *b = hash_table_get_instance(item, block_t, hash_link); - return b->lba == MERGE_LOUP32(key[0], key[1]); -} - -static void cache_remove_callback(link_t *item) -{ -} - -static hash_table_operations_t cache_ops = { +static size_t cache_key_hash(void *key) +{ + aoff64_t *lba = (aoff64_t*)key; + return *lba; +} + +static size_t cache_hash(const ht_link_t *item) +{ + block_t *b = hash_table_get_inst(item, block_t, hash_link); + return b->lba; +} + +static bool cache_key_equal(void *key, const ht_link_t *item) +{ + aoff64_t *lba = (aoff64_t*)key; + block_t *b = hash_table_get_inst(item, block_t, hash_link); + return b->lba == *lba; +} + + +static hash_table_ops_t cache_ops = { .hash = cache_hash, - .compare = cache_compare, - .remove_callback = cache_remove_callback + .key_hash = cache_key_hash, + .key_equal = cache_key_equal, + .equal = 0, + .remove_callback = 0 }; @@ -305,6 +310,5 @@ cache->blocks_cluster = cache->lblock_size / devcon->pblock_size; - if (!hash_table_create(&cache->block_hash, CACHE_BUCKETS, 2, - &cache_ops)) { + if (!hash_table_create(&cache->block_hash, 0, 0, &cache_ops)) { free(cache); return ENOMEM; @@ -344,9 +348,5 @@ } - unsigned long key[2] = { - LOWER32(b->lba), - UPPER32(b->lba) - }; - hash_table_remove(&cache->block_hash, key, 2); + hash_table_remove_item(&cache->block_hash, &b->hash_link); free(b->data); @@ -380,5 +380,4 @@ fibril_rwlock_initialize(&b->contents_lock); link_initialize(&b->free_link); - link_initialize(&b->hash_link); } @@ -400,9 +399,5 @@ cache_t *cache; block_t *b; - link_t *l; - unsigned long key[2] = { - LOWER32(ba), - UPPER32(ba) - }; + link_t *link; int rc; @@ -420,11 +415,11 @@ fibril_mutex_lock(&cache->lock); - l = hash_table_find(&cache->block_hash, key); - if (l) { + ht_link_t *hlink = hash_table_find(&cache->block_hash, &ba); + if (hlink) { found: /* * We found the block in the cache. */ - b = hash_table_get_instance(l, block_t, hash_link); + b = hash_table_get_inst(hlink, block_t, hash_link); fibril_mutex_lock(&b->lock); if (b->refcnt++ == 0) @@ -464,6 +459,6 @@ goto out; } - l = list_first(&cache->free_list); - b = list_get_instance(l, block_t, free_link); + link = list_first(&cache->free_list); + b = list_get_instance(link, block_t, free_link); fibril_mutex_lock(&b->lock); @@ -505,6 +500,6 @@ goto retry; } - l = hash_table_find(&cache->block_hash, key); - if (l) { + hlink = hash_table_find(&cache->block_hash, &ba); + if (hlink) { /* * Someone else must have already @@ -528,9 +523,5 @@ */ list_remove(&b->free_link); - unsigned long temp_key[2] = { - LOWER32(b->lba), - UPPER32(b->lba) - }; - hash_table_remove(&cache->block_hash, temp_key, 2); + hash_table_remove_item(&cache->block_hash, &b->hash_link); } @@ -540,5 +531,5 @@ b->lba = ba; b->pba = ba_ltop(devcon, b->lba); - hash_table_insert(&cache->block_hash, key, &b->hash_link); + hash_table_insert(&cache->block_hash, &b->hash_link); /* @@ -652,9 +643,5 @@ * Take the block out of the cache and free it. */ - unsigned long key[2] = { - LOWER32(block->lba), - UPPER32(block->lba) - }; - hash_table_remove(&cache->block_hash, key, 2); + hash_table_remove_item(&cache->block_hash, &block->hash_link); fibril_mutex_unlock(&block->lock); free(block->data); Index: uspace/lib/block/libblock.h =================================================================== --- uspace/lib/block/libblock.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/block/libblock.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -84,5 +84,5 @@ link_t free_link; /** Link for placing the block into the block hash table. */ - link_t hash_link; + ht_link_t hash_link; /** Buffer with the block data. */ void *data; Index: uspace/lib/c/Makefile =================================================================== --- uspace/lib/c/Makefile (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/Makefile (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -112,5 +112,4 @@ generic/adt/list.c \ generic/adt/hash_table.c \ - generic/adt/hash_set.c \ generic/adt/dynamic_fifo.c \ generic/adt/char_map.c \ Index: pace/lib/c/generic/adt/hash_set.c =================================================================== --- uspace/lib/c/generic/adt/hash_set.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ (revision ) @@ -1,382 +1,0 @@ -/* - * Copyright (c) 2008 Jakub Jermar - * Copyright (c) 2011 Radim Vansa - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup libc - * @{ - */ -/** @file - */ - -#include -#include -#include -#include -#include -#include - -/** Create chained hash set - * - * @param h Hash set structure to be initialized. - * @param[in] hash Hash function - * @param[in] equals Equals function - * @param[in] init_size Initial hash set size - * - * @return True on success - * - */ -int hash_set_init(hash_set_t *h, hash_set_hash hash, hash_set_equals equals, - size_t init_size) -{ - assert(h); - assert(hash); - assert(equals); - - if (init_size < HASH_SET_MIN_SIZE) - init_size = HASH_SET_MIN_SIZE; - - h->table = malloc(init_size * sizeof(link_t)); - if (!h->table) - return false; - - for (size_t i = 0; i < init_size; i++) - list_initialize(&h->table[i]); - - h->size = init_size; - h->count = 0; - h->hash = hash; - h->equals = equals; - - return true; -} - -/** Destroy a hash table instance. - * - * @param h Hash table to be destroyed. - * - */ -void hash_set_destroy(hash_set_t *h) -{ - assert(h); - free(h->table); -} - -/** Rehash the internal table to new table - * - * @param h Original hash set - * @param new_table Memory for the new table - * @param new_size Size of the new table - */ -static void hash_set_rehash(hash_set_t *h, list_t *new_table, - size_t new_size) -{ - assert(new_size >= HASH_SET_MIN_SIZE); - - for (size_t bucket = 0; bucket < new_size; bucket++) - list_initialize(&new_table[bucket]); - - for (size_t bucket = 0; bucket < h->size; bucket++) { - link_t *cur; - link_t *next; - - for (cur = h->table[bucket].head.next; - cur != &h->table[bucket].head; - cur = next) { - next = cur->next; - list_append(cur, &new_table[h->hash(cur) % new_size]); - } - } - - list_t *old_table = h->table; - h->table = new_table; - free(old_table); - h->size = new_size; -} - -/** Insert item into the set. - * - * If the set already contains equivalent object, - * the function fails. - * - * @param h Hash table. - * @param key Array of all keys necessary to compute hash index. - * @param item Item to be inserted into the hash table. - * - * @return True if the object was inserted - * @return Ffalse if the set already contained equivalent object. - * - */ -int hash_set_insert(hash_set_t *h, link_t *item) -{ - assert(item); - assert(h); - assert(h->hash); - assert(h->equals); - - unsigned long hash = h->hash(item); - unsigned long chain = hash % h->size; - - list_foreach(h->table[chain], cur) { - if (h->equals(cur, item)) - return false; - } - - if (h->count + 1 > h->size) { - size_t new_size = h->size * 2; - list_t *temp = malloc(new_size * sizeof(list_t)); - if (temp != NULL) - hash_set_rehash(h, temp, new_size); - - /* - * If the allocation fails, just use the same - * old table and try to rehash next time. - */ - chain = hash % h->size; - } - - h->count++; - list_append(item, &h->table[chain]); - - return true; -} - -/** Search the hash set for a matching object and return it - * - * @param h Hash set - * @param item The item that should equal to the matched object - * - * @return Matching item on success, NULL if there is no such item. - * - */ -link_t *hash_set_find(hash_set_t *h, const link_t *item) -{ - assert(h); - assert(h->hash); - assert(h->equals); - - unsigned long chain = h->hash(item) % h->size; - - list_foreach(h->table[chain], cur) { - if (h->equals(cur, item)) - return cur; - } - - return NULL; -} - -/** Remove first matching object from the hash set and return it - * - * @param h Hash set. - * @param item The item that should be equal to the matched object - * - * @return The removed item or NULL if this is not found. - * - */ -link_t *hash_set_remove(hash_set_t *h, const link_t *item) -{ - assert(h); - assert(h->hash); - assert(h->equals); - - link_t *cur = hash_set_find(h, item); - if (cur) { - list_remove(cur); - - h->count--; - if (4 * h->count < h->size && h->size > HASH_SET_MIN_SIZE) { - size_t new_size = h->size / 2; - if (new_size < HASH_SET_MIN_SIZE) - /* possible e.g. if init_size == HASH_SET_MIN_SIZE + 1 */ - new_size = HASH_SET_MIN_SIZE; - - list_t *temp = malloc(new_size * sizeof (list_t)); - if (temp != NULL) - hash_set_rehash(h, temp, new_size); - } - } - - return cur; -} - -/** Remove all elements for which the function returned non-zero - * - * The function can also destroy the element. - * - * @param h Hash set. - * @param f Function to be applied. - * @param arg Argument to be passed to the function. - * - */ -void hash_set_remove_selected(hash_set_t *h, int (*f)(link_t *, void *), - void *arg) -{ - assert(h); - assert(h->table); - - for (size_t bucket = 0; bucket < h->size; bucket++) { - link_t *prev = &h->table[bucket].head; - link_t *cur; - link_t *next; - - for (cur = h->table[bucket].head.next; - cur != &h->table[bucket].head; - cur = next) { - next = cur->next; - if (f(cur, arg)) { - prev->next = next; - next->prev = prev; - h->count--; - } else - prev = cur; - } - } - - if (4 * h->count < h->size && h->size > HASH_SET_MIN_SIZE) { - size_t new_size = h->size / 2; - if (new_size < HASH_SET_MIN_SIZE) - /* possible e.g. if init_size == HASH_SET_MIN_SIZE + 1 */ - new_size = HASH_SET_MIN_SIZE; - - list_t *temp = malloc(new_size * sizeof (list_t)); - if (temp != NULL) - hash_set_rehash(h, temp, new_size); - } -} - -/** Apply function to all items in hash set - * - * @param h Hash set. - * @param f Function to be applied. - * @param arg Argument to be passed to the function. - * - */ -void hash_set_apply(hash_set_t *h, void (*f)(link_t *, void *), void *arg) -{ - assert(h); - assert(h->table); - - for (size_t bucket = 0; bucket < h->size; bucket++) { - link_t *cur; - link_t *next; - - for (cur = h->table[bucket].head.next; - cur != &h->table[bucket].head; - cur = next) { - - /* - * The next pointer must be stored prior to the functor - * call to allow using destructor as the functor (the - * free function could overwrite the cur->next pointer). - */ - next = cur->next; - f(cur, arg); - } - } -} - -/** Remove all elements from the set. - * - * The table is reallocated to the minimum size. - * - * @param h Hash set - * @param f Function (destructor?) applied to all element. Can be NULL. - * @param arg Argument to the destructor. - * - */ -void hash_set_clear(hash_set_t *h, void (*f)(link_t *, void *), void *arg) -{ - assert(h); - assert(h->table); - - for (size_t bucket = 0; bucket < h->size; bucket++) { - link_t *cur; - link_t *next; - - for (cur = h->table[bucket].head.next; - cur != &h->table[bucket].head; - cur = next) { - next = cur->next; - list_remove(cur); - if (f != NULL) - f(cur, arg); - } - } - - assert(h->size >= HASH_SET_MIN_SIZE); - list_t *new_table = - realloc(h->table, HASH_SET_MIN_SIZE * sizeof(list_t)); - - /* We are shrinking, therefore we shouldn't get NULL */ - assert(new_table); - - if (h->table != new_table) { - /* Init the lists, pointers to itself are used in them */ - for (size_t bucket = 0; bucket < HASH_SET_MIN_SIZE; ++bucket) - list_initialize(&new_table[bucket]); - - h->table = new_table; - } - - h->count = 0; - h->size = HASH_SET_MIN_SIZE; -} - -/** Get hash set size - * - * @param hHash set - * - * @return Number of elements in the set. - * - */ -size_t hash_set_count(const hash_set_t *h) -{ - assert(h); - return h->count; -} - -/** Check whether element is contained in the hash set - * - * @param h Hash set - * @param item Item that should be equal to the matched object - * - * @return True if the hash set contains equal object - * @return False otherwise - * - */ -int hash_set_contains(const hash_set_t *h, const link_t *item) -{ - /* - * The hash_set_find cannot accept constant hash set, - * because we can modify the returned element. But in - * this case we are using it safely. - */ - return hash_set_find((hash_set_t *) h, item) != NULL; -} - -/** @} - */ Index: uspace/lib/c/generic/adt/hash_table.c =================================================================== --- uspace/lib/c/generic/adt/hash_table.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/generic/adt/hash_table.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -1,4 +1,6 @@ /* * Copyright (c) 2008 Jakub Jermar + * Copyright (c) 2012 Adam Hraska + * * All rights reserved. * @@ -34,5 +36,15 @@ /* - * This is an implementation of generic chained hash table. + * This is an implementation of a generic resizable chained hash table. + * + * The table grows to 2*n+1 buckets each time, starting at n == 89, + * per Thomas Wang's recommendation: + * http://www.concentric.net/~Ttwang/tech/hashsize.htm + * + * This policy produces prime table sizes for the first five resizes + * and generally produces table sizes which are either prime or + * have fairly large (prime/odd) divisors. Having a prime table size + * mitigates the use of suboptimal hash functions and distributes + * items over the whole table. */ @@ -44,38 +56,99 @@ #include +/* Optimal initial bucket count. See comment above. */ +#define HT_MIN_BUCKETS 89 +/* The table is resized when the average load per bucket exceeds this number. */ +#define HT_MAX_LOAD 2 + + +static size_t round_up_size(size_t size); +static bool alloc_table(size_t bucket_cnt, list_t **pbuckets); +static void clear_items(hash_table_t *h); +static void resize(hash_table_t *h, size_t new_bucket_cnt); +static void grow_if_needed(hash_table_t *h); +static void shrink_if_needed(hash_table_t *h); + +/* Dummy do nothing callback to invoke in place of remove_callback == NULL. */ +static void nop_remove_callback(ht_link_t *item) +{ + /* no-op */ +} + + /** Create chained hash table. * * @param h Hash table structure. Will be initialized by this call. - * @param m Number of hash table buckets. + * @param init_size Initial desired number of hash table buckets. Pass zero + * if you want the default initial size. * @param max_keys Maximal number of keys needed to identify an item. - * @param op Hash table operations structure. + * @param op Hash table operations structure. remove_callback() + * is optional and can be NULL if no action is to be taken + * upon removal. equal() is optional if and only if + * hash_table_insert_unique() will never be invoked. + * All other operations are mandatory. * * @return True on success * */ -bool hash_table_create(hash_table_t *h, hash_count_t m, hash_count_t max_keys, - hash_table_operations_t *op) +bool hash_table_create(hash_table_t *h, size_t init_size, size_t max_load, + hash_table_ops_t *op) { assert(h); - assert(op && op->hash && op->compare); - assert(max_keys > 0); - - h->entry = malloc(m * sizeof(list_t)); - if (!h->entry) + assert(op && op->hash && op->key_hash && op->key_equal); + + /* Check for compulsory ops. */ + if (!op || !op->hash || !op->key_hash || !op->key_equal) return false; - memset((void *) h->entry, 0, m * sizeof(list_t)); - - hash_count_t i; - for (i = 0; i < m; i++) - list_initialize(&h->entry[i]); - - h->entries = m; - h->max_keys = max_keys; + h->bucket_cnt = round_up_size(init_size); + + if (!alloc_table(h->bucket_cnt, &h->bucket)) + return false; + + h->max_load = (max_load == 0) ? HT_MAX_LOAD : max_load; + h->item_cnt = 0; h->op = op; + h->full_item_cnt = h->max_load * h->bucket_cnt; + h->apply_ongoing = false; + + if (h->op->remove_callback == 0) { + h->op->remove_callback = nop_remove_callback; + } return true; } +/** Destroy a hash table instance. + * + * @param h Hash table to be destroyed. + * + */ +void hash_table_destroy(hash_table_t *h) +{ + assert(h && h->bucket); + assert(!h->apply_ongoing); + + clear_items(h); + + free(h->bucket); + + h->bucket = 0; + h->bucket_cnt = 0; +} + +/** Returns true if there are no items in the table. */ +bool hash_table_empty(hash_table_t *h) +{ + assert(h && h->bucket); + return h->item_cnt == 0; +} + +/** Returns the number of items in the table. */ +size_t hash_table_size(hash_table_t *h) +{ + assert(h && h->bucket); + return h->item_cnt; +} + /** Remove all elements from the hash table * @@ -84,29 +157,32 @@ void hash_table_clear(hash_table_t *h) { - for (hash_count_t chain = 0; chain < h->entries; ++chain) { - link_t *cur; - link_t *next; - - for (cur = h->entry[chain].head.next; - cur != &h->entry[chain].head; - cur = next) { - next = cur->next; + assert(h && h->bucket); + assert(!h->apply_ongoing); + + clear_items(h); + + /* Shrink the table to its minimum size if possible. */ + if (HT_MIN_BUCKETS < h->bucket_cnt) { + resize(h, HT_MIN_BUCKETS); + } +} + +/** Unlinks and removes all items but does not resize. */ +static void clear_items(hash_table_t *h) +{ + if (h->item_cnt == 0) + return; + + for (size_t idx = 0; idx < h->bucket_cnt; ++idx) { + list_foreach_safe(h->bucket[idx], cur, next) { + assert(cur); + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + list_remove(cur); - h->op->remove_callback(cur); - } - } -} - -/** Destroy a hash table instance. - * - * @param h Hash table to be destroyed. - * - */ -void hash_table_destroy(hash_table_t *h) -{ - assert(h); - assert(h->entry); - - free(h->entry); + h->op->remove_callback(cur_link); + } + } + + h->item_cnt = 0; } @@ -117,13 +193,52 @@ * @param item Item to be inserted into the hash table. */ -void hash_table_insert(hash_table_t *h, unsigned long key[], link_t *item) +void hash_table_insert(hash_table_t *h, ht_link_t *item) { assert(item); - assert(h && h->op && h->op->hash && h->op->compare); - - hash_index_t chain = h->op->hash(key); - assert(chain < h->entries); - - list_append(item, &h->entry[chain]); + assert(h && h->bucket); + assert(!h->apply_ongoing); + + size_t idx = h->op->hash(item) % h->bucket_cnt; + + list_append(&item->link, &h->bucket[idx]); + ++h->item_cnt; + grow_if_needed(h); +} + + +/** Insert item into a hash table if not already present. + * + * @param h Hash table. + * @param key Array of all keys necessary to compute hash index. + * @param item Item to be inserted into the hash table. + * + * @return False if such an item had already been inserted. + * @return True if the inserted item was the only item with such a lookup key. + */ +bool hash_table_insert_unique(hash_table_t *h, ht_link_t *item) +{ + assert(item); + assert(h && h->bucket && h->bucket_cnt); + assert(h->op && h->op->hash && h->op->equal); + assert(!h->apply_ongoing); + + size_t idx = h->op->hash(item) % h->bucket_cnt; + + /* Check for duplicates. */ + list_foreach(h->bucket[idx], cur) { + /* + * We could filter out items using their hashes first, but + * calling equal() might very well be just as fast. + */ + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + if (h->op->equal(cur_link, item)) + return false; + } + + list_append(&item->link, &h->bucket[idx]); + ++h->item_cnt; + grow_if_needed(h); + + return true; } @@ -136,20 +251,45 @@ * */ -link_t *hash_table_find(hash_table_t *h, unsigned long key[]) -{ - assert(h && h->op && h->op->hash && h->op->compare); - - hash_index_t chain = h->op->hash(key); - assert(chain < h->entries); - - list_foreach(h->entry[chain], cur) { - if (h->op->compare(key, h->max_keys, cur)) { - /* - * The entry is there. - */ - return cur; - } - } - +ht_link_t *hash_table_find(const hash_table_t *h, void *key) +{ + assert(h && h->bucket); + + size_t idx = h->op->key_hash(key) % h->bucket_cnt; + + list_foreach(h->bucket[idx], cur) { + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + /* + * Is this is the item we are looking for? We could have first + * checked if the hashes match but op->key_equal() may very well be + * just as fast as op->hash(). + */ + if (h->op->key_equal(key, cur_link)) { + return cur_link; + } + } + + return NULL; +} + +/** Find the next item equal to item. */ +ht_link_t *hash_table_find_next(const hash_table_t *h, ht_link_t *item) +{ + assert(item); + assert(h && h->bucket); + + /* Traverse the circular list until we reach the starting item again. */ + for (link_t *cur = item->link.next; cur != &item->link; cur = cur->next) { + assert(cur); + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + /* + * Is this is the item we are looking for? We could have first + * checked if the hashes match but op->equal() may very well be + * just as fast as op->hash(). + */ + if (h->op->equal(cur_link, item)) { + return cur_link; + } + } + return NULL; } @@ -163,76 +303,170 @@ * the hash table. * @param keys Number of keys in the 'key' array. - * - */ -void hash_table_remove(hash_table_t *h, unsigned long key[], hash_count_t keys) -{ - assert(h && h->op && h->op->hash && h->op->compare && - h->op->remove_callback); - assert(keys <= h->max_keys); - - if (keys == h->max_keys) { - /* - * All keys are known, hash_table_find() can be used to find the - * entry. + * + * @return Returns the number of removed items. + */ +size_t hash_table_remove(hash_table_t *h, void *key) +{ + assert(h && h->bucket); + assert(!h->apply_ongoing); + + size_t idx = h->op->key_hash(key) % h->bucket_cnt; + + size_t removed = 0; + + list_foreach_safe(h->bucket[idx], cur, next) { + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + + if (h->op->key_equal(key, cur_link)) { + ++removed; + list_remove(cur); + h->op->remove_callback(cur_link); + } + } + + h->item_cnt -= removed; + shrink_if_needed(h); + + return removed; +} + +/** Removes an item already present in the table. The item must be in the table.*/ +void hash_table_remove_item(hash_table_t *h, ht_link_t *item) +{ + assert(item); + assert(h && h->bucket); + assert(link_in_use(&item->link)); + + list_remove(&item->link); + --h->item_cnt; + h->op->remove_callback(item); + shrink_if_needed(h); +} + +/** Apply function to all items in hash table. + * + * @param h Hash table. + * @param f Function to be applied. Return false if no more items + * should be visited. The functor may only delete the supplied + * item. It must not delete the successor of the item passed + * in the first argument. + * @param arg Argument to be passed to the function. + */ +void hash_table_apply(hash_table_t *h, bool (*f)(ht_link_t *, void *), void *arg) +{ + assert(f); + assert(h && h->bucket); + + if (h->item_cnt == 0) + return; + + h->apply_ongoing = true; + + for (size_t idx = 0; idx < h->bucket_cnt; ++idx) { + list_foreach_safe(h->bucket[idx], cur, next) { + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + /* + * The next pointer had already been saved. f() may safely + * delete cur (but not next!). + */ + if (!f(cur_link, arg)) + return; + } + } + + h->apply_ongoing = false; + + shrink_if_needed(h); + grow_if_needed(h); +} + +/** Rounds up size to the nearest suitable table size. */ +static size_t round_up_size(size_t size) +{ + size_t rounded_size = HT_MIN_BUCKETS; + + while (rounded_size < size) { + rounded_size = 2 * rounded_size + 1; + } + + return rounded_size; +} + +/** Allocates and initializes the desired number of buckets. True if successful.*/ +static bool alloc_table(size_t bucket_cnt, list_t **pbuckets) +{ + assert(pbuckets && HT_MIN_BUCKETS <= bucket_cnt); + + list_t *buckets = malloc(bucket_cnt * sizeof(list_t)); + if (!buckets) + return false; + + for (size_t i = 0; i < bucket_cnt; i++) + list_initialize(&buckets[i]); + + *pbuckets = buckets; + return true; +} + + +/** Shrinks the table if the table is only sparely populated. */ +static inline void shrink_if_needed(hash_table_t *h) +{ + if (h->item_cnt <= h->full_item_cnt / 4 && HT_MIN_BUCKETS < h->bucket_cnt) { + /* + * Keep the bucket_cnt odd (possibly also prime). + * Shrink from 2n + 1 to n. Integer division discards the +1. */ - - link_t *cur = hash_table_find(h, key); - if (cur) { - list_remove(cur); - h->op->remove_callback(cur); - } - + size_t new_bucket_cnt = h->bucket_cnt / 2; + resize(h, new_bucket_cnt); + } +} + +/** Grows the table if table load exceeds the maximum allowed. */ +static inline void grow_if_needed(hash_table_t *h) +{ + /* Grow the table if the average bucket load exceeds the maximum. */ + if (h->full_item_cnt < h->item_cnt) { + /* Keep the bucket_cnt odd (possibly also prime). */ + size_t new_bucket_cnt = 2 * h->bucket_cnt + 1; + resize(h, new_bucket_cnt); + } +} + +/** Allocates and rehashes items to a new table. Frees the old table. */ +static void resize(hash_table_t *h, size_t new_bucket_cnt) +{ + assert(h && h->bucket); + assert(HT_MIN_BUCKETS <= new_bucket_cnt); + + /* We are traversing the table and resizing would mess up the buckets. */ + if (h->apply_ongoing) return; - } - - /* - * Fewer keys were passed. - * Any partially matching entries are to be removed. - */ - hash_index_t chain; - for (chain = 0; chain < h->entries; chain++) { - for (link_t *cur = h->entry[chain].head.next; - cur != &h->entry[chain].head; - cur = cur->next) { - if (h->op->compare(key, keys, cur)) { - link_t *hlp; - - hlp = cur; - cur = cur->prev; - - list_remove(hlp); - h->op->remove_callback(hlp); - - continue; + + list_t *new_buckets; + + /* Leave the table as is if we cannot resize. */ + if (!alloc_table(new_bucket_cnt, &new_buckets)) + return; + + if (0 < h->item_cnt) { + /* Rehash all the items to the new table. */ + for (size_t old_idx = 0; old_idx < h->bucket_cnt; ++old_idx) { + list_foreach_safe(h->bucket[old_idx], cur, next) { + ht_link_t *cur_link = member_to_inst(cur, ht_link_t, link); + + size_t new_idx = h->op->hash(cur_link) % new_bucket_cnt; + list_remove(cur); + list_append(cur, &new_buckets[new_idx]); } } } -} - -/** Apply function to all items in hash table. - * - * @param h Hash table. - * @param f Function to be applied. - * @param arg Argument to be passed to the function. - * - */ -void hash_table_apply(hash_table_t *h, void (*f)(link_t *, void *), void *arg) -{ - for (hash_index_t bucket = 0; bucket < h->entries; bucket++) { - link_t *cur; - link_t *next; - - for (cur = h->entry[bucket].head.next; cur != &h->entry[bucket].head; - cur = next) { - /* - * The next pointer must be stored prior to the functor - * call to allow using destructor as the functor (the - * free function could overwrite the cur->next pointer). - */ - next = cur->next; - f(cur, arg); - } - } -} + + free(h->bucket); + h->bucket = new_buckets; + h->bucket_cnt = new_bucket_cnt; + h->full_item_cnt = h->max_load * h->bucket_cnt; +} + /** @} Index: uspace/lib/c/generic/async.c =================================================================== --- uspace/lib/c/generic/async.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/generic/async.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -115,6 +115,4 @@ #include -#define CLIENT_HASH_TABLE_BUCKETS 32 -#define CONN_HASH_TABLE_BUCKETS 32 /** Session data */ @@ -204,5 +202,5 @@ /* Client connection data */ typedef struct { - link_t link; + ht_link_t link; task_id_t in_task_id; @@ -216,5 +214,5 @@ /** Hash table link. */ - link_t link; + ht_link_t link; /** Incoming client task ID. */ @@ -392,31 +390,31 @@ static LIST_INITIALIZE(timeout_list); -static hash_index_t client_hash(unsigned long key[]) -{ - assert(key); - - return (((key[0]) >> 4) % CLIENT_HASH_TABLE_BUCKETS); -} - -static int client_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - assert(key); - assert(keys == 2); - assert(item); - - client_t *client = hash_table_get_instance(item, client_t, link); - return (key[0] == LOWER32(client->in_task_id) && - (key[1] == UPPER32(client->in_task_id))); -} - -static void client_remove(link_t *item) -{ -} +static size_t client_key_hash(void *k) +{ + task_id_t key = *(task_id_t*)k; + return key; +} + +static size_t client_hash(const ht_link_t *item) +{ + client_t *client = hash_table_get_inst(item, client_t, link); + return client_key_hash(&client->in_task_id); +} + +static bool client_key_equal(void *k, const ht_link_t *item) +{ + task_id_t key = *(task_id_t*)k; + client_t *client = hash_table_get_inst(item, client_t, link); + return key == client->in_task_id; +} + /** Operations for the client hash table. */ -static hash_table_operations_t client_hash_table_ops = { +static hash_table_ops_t client_hash_table_ops = { .hash = client_hash, - .compare = client_compare, - .remove_callback = client_remove + .key_hash = client_key_hash, + .key_equal = client_key_equal, + .equal = 0, + .remove_callback = 0 }; @@ -428,38 +426,31 @@ * */ -static hash_index_t conn_hash(unsigned long key[]) -{ - assert(key); - - return (((key[0]) >> 4) % CONN_HASH_TABLE_BUCKETS); -} - -/** Compare hash table item with a key. - * - * @param key Array containing the source phone hash as the only item. - * @param keys Expected 1 but ignored. - * @param item Connection hash table item. - * - * @return True on match, false otherwise. - * - */ -static int conn_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - assert(key); - assert(item); - - connection_t *conn = hash_table_get_instance(item, connection_t, link); - return (key[0] == conn->in_phone_hash); -} - -static void conn_remove(link_t *item) -{ -} +static size_t conn_key_hash(void *key) +{ + sysarg_t in_phone_hash = *(sysarg_t*)key; + return in_phone_hash ; +} + +static size_t conn_hash(const ht_link_t *item) +{ + connection_t *conn = hash_table_get_inst(item, connection_t, link); + return conn_key_hash(&conn->in_phone_hash); +} + +static bool conn_key_equal(void *key, const ht_link_t *item) +{ + sysarg_t in_phone_hash = *(sysarg_t*)key; + connection_t *conn = hash_table_get_inst(item, connection_t, link); + return (in_phone_hash == conn->in_phone_hash); +} + /** Operations for the connection hash table. */ -static hash_table_operations_t conn_hash_table_ops = { +static hash_table_ops_t conn_hash_table_ops = { .hash = conn_hash, - .compare = conn_compare, - .remove_callback = conn_remove + .key_hash = conn_key_hash, + .key_equal = conn_key_equal, + .equal = 0, + .remove_callback = 0 }; @@ -509,6 +500,5 @@ futex_down(&async_futex); - unsigned long key = call->in_phone_hash; - link_t *hlp = hash_table_find(&conn_hash_table, &key); + ht_link_t *hlp = hash_table_find(&conn_hash_table, &call->in_phone_hash); if (!hlp) { @@ -517,5 +507,5 @@ } - connection_t *conn = hash_table_get_instance(hlp, connection_t, link); + connection_t *conn = hash_table_get_inst(hlp, connection_t, link); msg_t *msg = malloc(sizeof(*msg)); @@ -697,14 +687,10 @@ static client_t *async_client_get(task_id_t client_id, bool create) { - unsigned long key[2] = { - LOWER32(client_id), - UPPER32(client_id), - }; client_t *client = NULL; futex_down(&async_futex); - link_t *lnk = hash_table_find(&client_hash_table, key); + ht_link_t *lnk = hash_table_find(&client_hash_table, &client_id); if (lnk) { - client = hash_table_get_instance(lnk, client_t, link); + client = hash_table_get_inst(lnk, client_t, link); atomic_inc(&client->refcnt); } else if (create) { @@ -715,5 +701,5 @@ atomic_set(&client->refcnt, 1); - hash_table_insert(&client_hash_table, key, &client->link); + hash_table_insert(&client_hash_table, &client->link); } } @@ -726,13 +712,9 @@ { bool destroy; - unsigned long key[2] = { - LOWER32(client->in_task_id), - UPPER32(client->in_task_id) - }; - + futex_down(&async_futex); if (atomic_predec(&client->refcnt) == 0) { - hash_table_remove(&client_hash_table, key, 2); + hash_table_remove(&client_hash_table, &client->in_task_id); destroy = true; } else @@ -830,6 +812,5 @@ */ futex_down(&async_futex); - unsigned long key = fibril_connection->in_phone_hash; - hash_table_remove(&conn_hash_table, &key, 1); + hash_table_remove(&conn_hash_table, &fibril_connection->in_phone_hash); futex_up(&async_futex); @@ -915,8 +896,7 @@ /* Add connection to the connection hash table */ - unsigned long key = conn->in_phone_hash; futex_down(&async_futex); - hash_table_insert(&conn_hash_table, &key, &conn->link); + hash_table_insert(&conn_hash_table, &conn->link); futex_up(&async_futex); @@ -1110,10 +1090,8 @@ void __async_init(void) { - if (!hash_table_create(&client_hash_table, CLIENT_HASH_TABLE_BUCKETS, - 2, &client_hash_table_ops)) + if (!hash_table_create(&client_hash_table, 0, 0, &client_hash_table_ops)) abort(); - if (!hash_table_create(&conn_hash_table, CONN_HASH_TABLE_BUCKETS, - 1, &conn_hash_table_ops)) + if (!hash_table_create(&conn_hash_table, 0, 0, &conn_hash_table_ops)) abort(); Index: uspace/lib/c/include/adt/hash.h =================================================================== --- uspace/lib/c/include/adt/hash.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) +++ uspace/lib/c/include/adt/hash.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup genericadt + * @{ + */ +/** @file + */ +#ifndef KERN_HASH_H_ +#define KERN_HASH_H_ + +#include + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline uint32_t hash_mix32(uint32_t hash) +{ + /* + * Thomas Wang's modification of Bob Jenkin's hash mixing function: + * http://www.concentric.net/~Ttwang/tech/inthash.htm + * Public domain. + */ + hash = ~hash + (hash << 15); + hash = hash ^ (hash >> 12); + hash = hash + (hash << 2); + hash = hash ^ (hash >> 4); + hash = hash * 2057; + hash = hash ^ (hash >> 16); + return hash; +} + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline uint64_t hash_mix64(uint64_t hash) +{ + /* + * Thomas Wang's public domain 64-bit hash mixing function: + * http://www.concentric.net/~Ttwang/tech/inthash.htm + */ + hash = (hash ^ 61) ^ (hash >> 16); + hash = hash + (hash << 3); + hash = hash ^ (hash >> 4); + hash = hash * 0x27d4eb2d; + hash = hash ^ (hash >> 15); + /* + * Lower order bits are mixed more thoroughly. Swap them with + * the higher order bits and make the resulting higher order bits + * more usable. + */ + return (hash << 32) | (hash >> 32); +} + +/** Produces a uniform hash affecting all output bits from the skewed input. */ +static inline size_t hash_mix(size_t hash) +{ +#ifdef __32_BITS__ + return hash_mix32(hash); +#elif defined(__64_BITS__) + return hash_mix64(hash); +#else +#error Unknown size_t size - cannot select proper hash mix function. +#endif +} + +/** Use to create a hash from multiple values. + * + * Typical usage: + * @code + * int car_id; + * bool car_convertible; + * // .. + * size_t hash = 0; + * hash = hash_combine(hash, car_id); + * hash = hash_combine(hash, car_convertible); + * // Now use hash as a hash of both car_id and car_convertible. + * @endcode + */ +static inline size_t hash_combine(size_t seed, size_t hash) +{ + /* + * todo: use Bob Jenkin's proper mixing hash pass: + * http://burtleburtle.net/bob/c/lookup3.c + */ + seed ^= hash + 0x9e3779b9 + + ((seed << 5) | (seed >> (sizeof(size_t) * 8 - 5))); + return seed; +} + +#endif Index: pace/lib/c/include/adt/hash_set.h =================================================================== --- uspace/lib/c/include/adt/hash_set.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ (revision ) @@ -1,84 +1,0 @@ -/* - * Copyright (c) 2006 Jakub Jermar - * Copyright (c) 2011 Radim Vansa - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup libc - * @{ - */ -/** @file - */ - -#ifndef LIBC_HASH_SET_H_ -#define LIBC_HASH_SET_H_ - -#include -#include - -#define HASH_SET_MIN_SIZE 8 - -typedef unsigned long (*hash_set_hash)(const link_t *); -typedef int (*hash_set_equals)(const link_t *, const link_t *); - -/** Hash table structure. */ -typedef struct { - list_t *table; - - /** Current table size */ - size_t size; - - /** - * Current number of entries. If count > size, - * the table is rehashed into table with double - * size. If (4 * count < size) && (size > min_size), - * the table is rehashed into table with half the size. - */ - size_t count; - - /** Hash function */ - hash_set_hash hash; - - /** Hash table item equals function */ - hash_set_equals equals; -} hash_set_t; - -extern int hash_set_init(hash_set_t *, hash_set_hash, hash_set_equals, size_t); -extern int hash_set_insert(hash_set_t *, link_t *); -extern link_t *hash_set_find(hash_set_t *, const link_t *); -extern int hash_set_contains(const hash_set_t *, const link_t *); -extern size_t hash_set_count(const hash_set_t *); -extern link_t *hash_set_remove(hash_set_t *, const link_t *); -extern void hash_set_remove_selected(hash_set_t *, - int (*)(link_t *, void *), void *); -extern void hash_set_destroy(hash_set_t *); -extern void hash_set_apply(hash_set_t *, void (*)(link_t *, void *), void *); -extern void hash_set_clear(hash_set_t *, void (*)(link_t *, void *), void *); - -#endif - -/** @} - */ Index: uspace/lib/c/include/adt/hash_table.h =================================================================== --- uspace/lib/c/include/adt/hash_table.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/include/adt/hash_table.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -1,4 +1,6 @@ /* * Copyright (c) 2006 Jakub Jermar + * Copyright (c) 2012 Adam Hraska + * * All rights reserved. * @@ -39,58 +41,65 @@ #include #include +#include -typedef unsigned long hash_count_t; -typedef unsigned long hash_index_t; +/** Opaque hash table link type. */ +typedef struct ht_link { + link_t link; +} ht_link_t; /** Set of operations for hash table. */ typedef struct { - /** Hash function. - * - * @param key Array of keys needed to compute hash index. - * All keys must be passed. - * - * @return Index into hash table. - * - */ - hash_index_t (*hash)(unsigned long key[]); + /** Returns the hash of the key stored in the item (ie its lookup key). */ + size_t (*hash)(const ht_link_t *item); - /** Hash table item comparison function. - * - * @param key Array of keys that will be compared with item. It is - * not necessary to pass all keys. - * - * @return True if the keys match, false otherwise. - * - */ - int (*compare)(unsigned long key[], hash_count_t keys, link_t *item); + /** Returns the hash of the key. */ + size_t (*key_hash)(void *key); + /** True if the items are equal (have the same lookup keys). */ + bool (*equal)(const ht_link_t *item1, const ht_link_t *item2); + + /** Returns true if the key is equal to the item's lookup key. */ + bool (*key_equal)(void *key, const ht_link_t *item); + /** Hash table item removal callback. + * + * Must not invoke any mutating functions of the hash table. * * @param item Item that was removed from the hash table. - * */ - void (*remove_callback)(link_t *item); -} hash_table_operations_t; + void (*remove_callback)(ht_link_t *item); +} hash_table_ops_t; /** Hash table structure. */ typedef struct { - list_t *entry; - hash_count_t entries; - hash_count_t max_keys; - hash_table_operations_t *op; + hash_table_ops_t *op; + list_t *bucket; + size_t bucket_cnt; + size_t full_item_cnt; + size_t item_cnt; + size_t max_load; + bool apply_ongoing; } hash_table_t; -#define hash_table_get_instance(item, type, member) \ - list_get_instance((item), type, member) +#define hash_table_get_inst(item, type, member) \ + member_to_inst((item), type, member) -extern bool hash_table_create(hash_table_t *, hash_count_t, hash_count_t, - hash_table_operations_t *); +extern bool hash_table_create(hash_table_t *, size_t, size_t, + hash_table_ops_t *); +extern void hash_table_destroy(hash_table_t *); + +extern bool hash_table_empty(hash_table_t *); +extern size_t hash_table_size(hash_table_t *); + extern void hash_table_clear(hash_table_t *); -extern void hash_table_insert(hash_table_t *, unsigned long [], link_t *); -extern link_t *hash_table_find(hash_table_t *, unsigned long []); -extern void hash_table_remove(hash_table_t *, unsigned long [], hash_count_t); -extern void hash_table_destroy(hash_table_t *); -extern void hash_table_apply(hash_table_t *, void (*)(link_t *, void *), - void *); +extern void hash_table_insert(hash_table_t *, ht_link_t *); +extern bool hash_table_insert_unique(hash_table_t *, ht_link_t *); +extern ht_link_t *hash_table_find(const hash_table_t *, void *); +extern ht_link_t *hash_table_find_next(const hash_table_t *, ht_link_t *); +extern size_t hash_table_remove(hash_table_t *, void *); +extern void hash_table_remove_item(hash_table_t *, ht_link_t *); +extern void hash_table_apply(hash_table_t *, bool (*)(ht_link_t *, void *), + void *); + #endif Index: uspace/lib/c/include/adt/list.h =================================================================== --- uspace/lib/c/include/adt/list.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/include/adt/list.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -71,6 +71,43 @@ iterator != &(list).head; iterator = iterator->next) +/** Unlike list_foreach(), allows removing items while traversing a list. + * + * @code + * list_t mylist; + * typedef struct item { + * int value; + * link_t item_link; + * } item_t; + * + * //.. + * + * // Print each list element's value and remove the element from the list. + * list_foreach_safe(mylist, cur_link, next_link) { + * item_t *cur_item = list_get_instance(cur_link, item_t, item_link); + * printf("%d\n", cur_item->value); + * list_remove(cur_link); + * } + * @endcode + * + * @param list List to traverse. + * @param iterator Iterator to the current element of the list. + * The item this iterator points may be safely removed + * from the list. + * @param next_iter Iterator to the next element of the list. + */ +#define list_foreach_safe(list, iterator, next_iter) \ + for (link_t *iterator = (list).head.next, \ + *next_iter = iterator->next; \ + iterator != &(list).head; \ + iterator = next_iter, next_iter = iterator->next) + #define assert_link_not_used(link) \ assert(((link)->prev == NULL) && ((link)->next == NULL)) + +/** Returns true if the link is definitely part of a list. False if not sure. */ +static inline int link_in_use(link_t *link) +{ + return link->prev != NULL && link->next != NULL; +} /** Initialize doubly-linked circular list link Index: uspace/lib/c/include/macros.h =================================================================== --- uspace/lib/c/include/macros.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/c/include/macros.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -52,4 +52,10 @@ | ((((uint64_t) (up)) & 0xffffffff) << 32)) +#ifndef member_to_inst +#define member_to_inst(ptr_member, type, member_identif) \ + ((type*) (((void*)(ptr_member)) - ((void*)&(((type*)0)->member_identif)))) +#endif + + #endif Index: uspace/lib/nic/include/nic.h =================================================================== --- uspace/lib/nic/include/nic.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/nic/include/nic.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -40,4 +40,5 @@ #include +#include #include #include @@ -53,5 +54,5 @@ */ typedef struct nic_wol_virtue { - link_t item; + ht_link_t item; nic_wv_id_t id; nic_wv_type_t type; Index: uspace/lib/nic/include/nic_addr_db.h =================================================================== --- uspace/lib/nic/include/nic_addr_db.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/nic/include/nic_addr_db.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -43,14 +43,5 @@ #endif -#include - -/** - * Initial size of DB's hash set - */ -#define NIC_ADDR_DB_INIT_SIZE 8 -/** - * Maximal length of addresses in the DB (in bytes). - */ -#define NIC_ADDR_MAX_LENGTH 16 +#include /** @@ -58,25 +49,14 @@ */ typedef struct nic_addr_db { - hash_set_t set; + hash_table_t set; size_t addr_len; } nic_addr_db_t; -/** - * Helper structure for keeping the address in the hash set. - */ -typedef struct nic_addr_entry { - link_t item; - size_t addr_len; - uint8_t addr[NIC_ADDR_MAX_LENGTH]; -} nic_addr_entry_t; extern int nic_addr_db_init(nic_addr_db_t *db, size_t addr_len); extern void nic_addr_db_clear(nic_addr_db_t *db); extern void nic_addr_db_destroy(nic_addr_db_t *db); -extern size_t nic_addr_db_count(const nic_addr_db_t *db); extern int nic_addr_db_insert(nic_addr_db_t *db, const uint8_t *addr); extern int nic_addr_db_remove(nic_addr_db_t *db, const uint8_t *addr); -extern void nic_addr_db_remove_selected(nic_addr_db_t *db, - int (*func)(const uint8_t *, void *), void *arg); extern int nic_addr_db_contains(const nic_addr_db_t *db, const uint8_t *addr); extern void nic_addr_db_foreach(const nic_addr_db_t *db, Index: uspace/lib/nic/include/nic_wol_virtues.h =================================================================== --- uspace/lib/nic/include/nic_wol_virtues.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/nic/include/nic_wol_virtues.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -51,5 +51,5 @@ * Operations for table */ - hash_table_operations_t table_operations; + hash_table_ops_t table_operations; /** * WOL virtues hashed by their ID's. Index: uspace/lib/nic/src/nic_addr_db.c =================================================================== --- uspace/lib/nic/src/nic_addr_db.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/nic/src/nic_addr_db.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -35,4 +35,6 @@ * @brief Generic hash-set based database of addresses */ +#include "nic_addr_db.h" +#include "libarch/common.h" #include #include @@ -40,49 +42,72 @@ #include #include - -#include "nic_addr_db.h" - -/** - * Hash set helper function - */ -static int nic_addr_equals(const link_t *item1, const link_t *item2) -{ - assert(item1 && item2); - size_t addr_len = ((const nic_addr_entry_t *) item1)->addr_len; - - assert(addr_len == ((const nic_addr_entry_t *) item2)->addr_len); - - size_t i; - for (i = 0; i < addr_len; ++i) { - if (((nic_addr_entry_t *) item1)->addr[i] != - ((nic_addr_entry_t *) item2)->addr[i]) - return false; +#include +#include +#include + + +/** + * Helper structure for keeping the address in the hash set. + */ +typedef struct nic_addr_entry { + ht_link_t link; + uint8_t len; + uint8_t addr[1]; +} nic_addr_entry_t; + + +/* + * Hash table helper functions + */ +typedef struct { + size_t len; + const uint8_t *addr; +} addr_key_t; + +static bool nic_addr_key_equal(void *key_arg, const ht_link_t *item) +{ + addr_key_t *key = (addr_key_t*)key_arg; + nic_addr_entry_t *entry = member_to_inst(item, nic_addr_entry_t, link); + + return 0 == bcmp(entry->addr, key->addr, entry->len); +} + +static size_t addr_hash(size_t len, const uint8_t *addr) +{ + size_t hash = 0; + + for (size_t i = 0; i < len; ++i) { + hash = (hash << 5) ^ addr[i]; } - return true; -} - -/** - * Hash set helper function - */ -static unsigned long nic_addr_hash(const link_t *item) -{ - assert(item); - const nic_addr_entry_t *entry = (const nic_addr_entry_t *) item; - unsigned long hash = 0; - - size_t i; - for (i = 0; i < entry->addr_len; ++i) { - hash = (hash << 8) ^ (hash >> 24) ^ entry->addr[i]; - } + return hash; } -/** - * Helper wrapper - */ -static void nic_addr_destroy(link_t *item, void *unused) -{ - free(item); -} +static size_t nic_addr_key_hash(void *k) +{ + addr_key_t *key = (addr_key_t*)k; + return addr_hash(key->len, key->addr); +} + +static size_t nic_addr_hash(const ht_link_t *item) +{ + nic_addr_entry_t *entry = member_to_inst(item, nic_addr_entry_t, link); + return addr_hash(entry->len, entry->addr); +} + +static void nic_addr_removed(ht_link_t *item) +{ + nic_addr_entry_t *entry = member_to_inst(item, nic_addr_entry_t, link); + + free(entry); +} + +static hash_table_ops_t set_ops = { + .hash = nic_addr_hash, + .key_hash = nic_addr_key_hash, + .key_equal = nic_addr_key_equal, + .equal = 0, + .remove_callback = nic_addr_removed +}; /** @@ -99,11 +124,11 @@ { assert(db); - if (addr_len > NIC_ADDR_MAX_LENGTH) { + + if (addr_len > UCHAR_MAX) return EINVAL; - } - if (!hash_set_init(&db->set, nic_addr_hash, nic_addr_equals, - NIC_ADDR_DB_INIT_SIZE)) { + + if (!hash_table_create(&db->set, 0, 0, &set_ops)) return ENOMEM; - } + db->addr_len = addr_len; return EOK; @@ -118,5 +143,5 @@ { assert(db); - hash_set_clear(&db->set, nic_addr_destroy, NULL); + hash_table_clear(&db->set); } @@ -129,20 +154,7 @@ { assert(db); - hash_set_apply(&db->set, nic_addr_destroy, NULL); - hash_set_destroy(&db->set); -} - -/** - * Get number of addresses in the db - * - * @param db - * - * @return Number of adresses - */ -size_t nic_addr_db_count(const nic_addr_db_t *db) -{ - assert(db); - return hash_set_count(&db->set); -} + hash_table_destroy(&db->set); +} + /** @@ -160,12 +172,22 @@ { assert(db && addr); - nic_addr_entry_t *entry = malloc(sizeof (nic_addr_entry_t)); - if (entry == NULL) { + + addr_key_t key = { + .len = db->addr_len, + .addr = addr + }; + + if (hash_table_find(&db->set, &key)) + return EEXIST; + + nic_addr_entry_t *entry = malloc(sizeof(nic_addr_entry_t) + db->addr_len - 1); + if (entry == NULL) return ENOMEM; - } - entry->addr_len = db->addr_len; + + entry->len = (uint8_t) db->addr_len; memcpy(entry->addr, addr, db->addr_len); - - return hash_set_insert(&db->set, &entry->item) ? EOK : EEXIST; + + hash_table_insert(&db->set, &entry->link); + return EOK; } @@ -182,11 +204,14 @@ { assert(db && addr); - nic_addr_entry_t entry; - entry.addr_len = db->addr_len; - memcpy(entry.addr, addr, db->addr_len); - - link_t *removed = hash_set_remove(&db->set, &entry.item); - free(removed); - return removed != NULL ? EOK : ENOENT; + + addr_key_t key = { + .len = db->addr_len, + .addr = addr + }; + + if (hash_table_remove(&db->set, &key)) + return EOK; + else + return ENOENT; } @@ -202,9 +227,11 @@ { assert(db && addr); - nic_addr_entry_t entry; - entry.addr_len = db->addr_len; - memcpy(entry.addr, addr, db->addr_len); - - return hash_set_contains(&db->set, &entry.item); + + addr_key_t key = { + .len = db->addr_len, + .addr = addr + }; + + return 0 != hash_table_find(&db->set, &key); } @@ -220,7 +247,10 @@ * Helper function for nic_addr_db_foreach */ -static void nic_addr_db_fe_helper(link_t *item, void *arg) { +static bool nic_addr_db_fe_helper(ht_link_t *item, void *arg) +{ nic_addr_db_fe_arg_t *hs = (nic_addr_db_fe_arg_t *) arg; - hs->func(((nic_addr_entry_t *) item)->addr, hs->arg); + nic_addr_entry_t *entry = member_to_inst(item, nic_addr_entry_t, link); + hs->func(entry->addr, hs->arg); + return true; } @@ -237,39 +267,5 @@ { nic_addr_db_fe_arg_t hs = { .func = func, .arg = arg }; - hash_set_apply((hash_set_t *) &db->set, nic_addr_db_fe_helper, &hs); -} - -/** - * Helper structure for nic_addr_db_remove_selected - */ -typedef struct { - int (*func)(const uint8_t *, void *); - void *arg; -} nic_addr_db_rs_arg_t; - -/** - * Helper function for nic_addr_db_foreach - */ -static int nic_addr_db_rs_helper(link_t *item, void *arg) { - nic_addr_db_rs_arg_t *hs = (nic_addr_db_rs_arg_t *) arg; - int retval = hs->func(((nic_addr_entry_t *) item)->addr, hs->arg); - if (retval) { - free(item); - } - return retval; -} - -/** - * Removes all addresses for which the function returns non-zero. - * - * @param db - * @param func User-defined function - * @param arg Custom argument passed to the function - */ -void nic_addr_db_remove_selected(nic_addr_db_t *db, - int (*func)(const uint8_t *, void *), void *arg) -{ - nic_addr_db_rs_arg_t hs = { .func = func, .arg = arg }; - hash_set_remove_selected(&db->set, nic_addr_db_rs_helper, &hs); + hash_table_apply((hash_table_t*)&db->set, nic_addr_db_fe_helper, &hs); } Index: uspace/lib/nic/src/nic_wol_virtues.c =================================================================== --- uspace/lib/nic/src/nic_wol_virtues.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/lib/nic/src/nic_wol_virtues.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -37,32 +37,28 @@ #include "nic_wol_virtues.h" +#include "nic.h" #include #include -#define NIC_WV_HASH_COUNT 32 - -/** - * Hash table helper function - */ -static int nic_wv_compare(unsigned long key[], hash_count_t keys, - link_t *item) + +/* + * Hash table helper functions + */ + +static size_t nic_wv_key_hash(void *key) +{ + return *(nic_wv_id_t*) key; +} + +static size_t nic_wv_hash(const ht_link_t *item) { nic_wol_virtue_t *virtue = (nic_wol_virtue_t *) item; - return (virtue->id == (nic_wv_id_t) key[0]); -} - -/** - * Hash table helper function - */ -static void nic_wv_rc(link_t *item) -{ -} - -/** - * Hash table helper function - */ -static hash_index_t nic_wv_hash(unsigned long keys[]) -{ - return keys[0] % NIC_WV_HASH_COUNT; + return virtue->id; +} + +static bool nic_wv_key_equal(void *key, const ht_link_t *item) +{ + nic_wol_virtue_t *virtue = (nic_wol_virtue_t *) item; + return (virtue->id == *(nic_wv_id_t*) key); } @@ -77,10 +73,12 @@ int nic_wol_virtues_init(nic_wol_virtues_t *wvs) { - bzero(wvs, sizeof (nic_wol_virtues_t)); - wvs->table_operations.compare = nic_wv_compare; + bzero(wvs, sizeof(nic_wol_virtues_t)); wvs->table_operations.hash = nic_wv_hash; - wvs->table_operations.remove_callback = nic_wv_rc; - if (!hash_table_create(&wvs->table, NIC_WV_HASH_COUNT, 1, - &wvs->table_operations)) { + wvs->table_operations.key_hash = nic_wv_key_hash; + wvs->table_operations.key_equal = nic_wv_key_equal; + wvs->table_operations.equal = 0; + wvs->table_operations.remove_callback = 0; + + if (!hash_table_create(&wvs->table, 0, 0, &wvs->table_operations)) { return ENOMEM; } @@ -170,8 +168,6 @@ do { virtue->id = wvs->next_id++; - } while (NULL != - hash_table_find(&wvs->table, (unsigned long *) &virtue->id)); - hash_table_insert(&wvs->table, - (unsigned long *) &virtue->id, &virtue->item); + } while (NULL != hash_table_find(&wvs->table, &virtue->id)); + hash_table_insert(&wvs->table, &virtue->item); virtue->next = wvs->lists[virtue->type]; wvs->lists[virtue->type] = virtue; @@ -191,6 +187,6 @@ nic_wol_virtue_t *nic_wol_virtues_remove(nic_wol_virtues_t *wvs, nic_wv_id_t id) { - nic_wol_virtue_t *virtue = (nic_wol_virtue_t *) - hash_table_find(&wvs->table, (unsigned long *) &id); + nic_wol_virtue_t *virtue = + (nic_wol_virtue_t *) hash_table_find(&wvs->table, &id); if (virtue == NULL) { return NULL; @@ -198,5 +194,5 @@ /* Remove from filter_table */ - hash_table_remove(&wvs->table, (unsigned long *) &id, 1); + hash_table_remove_item(&wvs->table, &virtue->item); /* Remove from filter_types */ @@ -235,6 +231,5 @@ * constant virtue the retyping is correct. */ - link_t *virtue = hash_table_find( - &((nic_wol_virtues_t *) wvs)->table, (unsigned long *) &id); + ht_link_t *virtue = hash_table_find(&((nic_wol_virtues_t *) wvs)->table, &id); return (const nic_wol_virtue_t *) virtue; } Index: uspace/srv/devman/devman.c =================================================================== --- uspace/srv/devman/devman.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/devman/devman.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -66,50 +66,78 @@ /* hash table operations */ -static hash_index_t devices_hash(unsigned long key[]) -{ - return key[0] % DEVICE_BUCKETS; -} - -static int devman_devices_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - dev_node_t *dev = hash_table_get_instance(item, dev_node_t, devman_dev); - return (dev->handle == (devman_handle_t) key[0]); -} - -static int devman_functions_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - fun_node_t *fun = hash_table_get_instance(item, fun_node_t, devman_fun); - return (fun->handle == (devman_handle_t) key[0]); -} - -static int loc_functions_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - fun_node_t *fun = hash_table_get_instance(item, fun_node_t, loc_fun); - return (fun->service_id == (service_id_t) key[0]); -} - -static void devices_remove_callback(link_t *item) -{ -} - -static hash_table_operations_t devman_devices_ops = { - .hash = devices_hash, - .compare = devman_devices_compare, - .remove_callback = devices_remove_callback +static inline size_t handle_key_hash(void *key) +{ + devman_handle_t handle = *(devman_handle_t*)key; + return handle; +} + +static size_t devman_devices_hash(const ht_link_t *item) +{ + dev_node_t *dev = hash_table_get_inst(item, dev_node_t, devman_dev); + return handle_key_hash(&dev->handle); +} + +static size_t devman_functions_hash(const ht_link_t *item) +{ + fun_node_t *fun = hash_table_get_inst(item, fun_node_t, devman_fun); + return handle_key_hash(&fun->handle); +} + +static bool devman_devices_key_equal(void *key, const ht_link_t *item) +{ + devman_handle_t handle = *(devman_handle_t*)key; + dev_node_t *dev = hash_table_get_inst(item, dev_node_t, devman_dev); + return dev->handle == handle; +} + +static bool devman_functions_key_equal(void *key, const ht_link_t *item) +{ + devman_handle_t handle = *(devman_handle_t*)key; + fun_node_t *fun = hash_table_get_inst(item, fun_node_t, devman_fun); + return fun->handle == handle; +} + +static inline size_t service_id_key_hash(void *key) +{ + service_id_t service_id = *(service_id_t*)key; + return service_id; +} + +static size_t loc_functions_hash(const ht_link_t *item) +{ + fun_node_t *fun = hash_table_get_inst(item, fun_node_t, loc_fun); + return service_id_key_hash(&fun->service_id); +} + +static bool loc_functions_key_equal(void *key, const ht_link_t *item) +{ + service_id_t service_id = *(service_id_t*)key; + fun_node_t *fun = hash_table_get_inst(item, fun_node_t, loc_fun); + return fun->service_id == service_id; +} + + +static hash_table_ops_t devman_devices_ops = { + .hash = devman_devices_hash, + .key_hash = handle_key_hash, + .key_equal = devman_devices_key_equal, + .equal = 0, + .remove_callback = 0 }; -static hash_table_operations_t devman_functions_ops = { - .hash = devices_hash, - .compare = devman_functions_compare, - .remove_callback = devices_remove_callback +static hash_table_ops_t devman_functions_ops = { + .hash = devman_functions_hash, + .key_hash = handle_key_hash, + .key_equal = devman_functions_key_equal, + .equal = 0, + .remove_callback = 0 }; -static hash_table_operations_t loc_devices_ops = { - .hash = devices_hash, - .compare = loc_functions_compare, - .remove_callback = devices_remove_callback +static hash_table_ops_t loc_devices_ops = { + .hash = loc_functions_hash, + .key_hash = service_id_key_hash, + .key_equal = loc_functions_key_equal, + .equal = 0, + .remove_callback = 0 }; @@ -974,10 +1002,7 @@ tree->current_handle = 0; - hash_table_create(&tree->devman_devices, DEVICE_BUCKETS, 1, - &devman_devices_ops); - hash_table_create(&tree->devman_functions, DEVICE_BUCKETS, 1, - &devman_functions_ops); - hash_table_create(&tree->loc_functions, DEVICE_BUCKETS, 1, - &loc_devices_ops); + hash_table_create(&tree->devman_devices, 0, 0, &devman_devices_ops); + hash_table_create(&tree->devman_functions, 0, 0, &devman_functions_ops); + hash_table_create(&tree->loc_functions, 0, 0, &loc_devices_ops); fibril_rwlock_initialize(&tree->rwlock); @@ -1013,5 +1038,4 @@ list_initialize(&dev->functions); link_initialize(&dev->driver_devices); - link_initialize(&dev->devman_dev); return dev; @@ -1061,14 +1085,11 @@ dev_node_t *find_dev_node_no_lock(dev_tree_t *tree, devman_handle_t handle) { - unsigned long key = handle; - link_t *link; - assert(fibril_rwlock_is_locked(&tree->rwlock)); - link = hash_table_find(&tree->devman_devices, &key); + ht_link_t *link = hash_table_find(&tree->devman_devices, &handle); if (link == NULL) return NULL; - return hash_table_get_instance(link, dev_node_t, devman_dev); + return hash_table_get_inst(link, dev_node_t, devman_dev); } @@ -1144,6 +1165,4 @@ link_initialize(&fun->dev_functions); list_initialize(&fun->match_ids.ids); - link_initialize(&fun->devman_fun); - link_initialize(&fun->loc_fun); return fun; @@ -1194,15 +1213,13 @@ fun_node_t *find_fun_node_no_lock(dev_tree_t *tree, devman_handle_t handle) { - unsigned long key = handle; - link_t *link; fun_node_t *fun; assert(fibril_rwlock_is_locked(&tree->rwlock)); - link = hash_table_find(&tree->devman_functions, &key); + ht_link_t *link = hash_table_find(&tree->devman_functions, &handle); if (link == NULL) return NULL; - fun = hash_table_get_instance(link, fun_node_t, devman_fun); + fun = hash_table_get_inst(link, fun_node_t, devman_fun); return fun; @@ -1282,6 +1299,5 @@ /* Add the node to the handle-to-node map. */ dev->handle = ++tree->current_handle; - unsigned long key = dev->handle; - hash_table_insert(&tree->devman_devices, &key, &dev->devman_dev); + hash_table_insert(&tree->devman_devices, &dev->devman_dev); /* Add the node to the list of its parent's children. */ @@ -1304,6 +1320,5 @@ /* Remove node from the handle-to-node map. */ - unsigned long key = dev->handle; - hash_table_remove(&tree->devman_devices, &key, 1); + hash_table_remove(&tree->devman_devices, &dev->handle); /* Unlink from parent function. */ @@ -1346,6 +1361,5 @@ /* Add the node to the handle-to-node map. */ fun->handle = ++tree->current_handle; - unsigned long key = fun->handle; - hash_table_insert(&tree->devman_functions, &key, &fun->devman_fun); + hash_table_insert(&tree->devman_functions, &fun->devman_fun); /* Add the node to the list of its parent's children. */ @@ -1367,6 +1381,5 @@ /* Remove the node from the handle-to-node map. */ - unsigned long key = fun->handle; - hash_table_remove(&tree->devman_functions, &key, 1); + hash_table_remove(&tree->devman_functions, &fun->handle); /* Remove the node from the list of its parent's children. */ @@ -1481,11 +1494,9 @@ { fun_node_t *fun = NULL; - link_t *link; - unsigned long key = (unsigned long) service_id; fibril_rwlock_read_lock(&tree->rwlock); - link = hash_table_find(&tree->loc_functions, &key); + ht_link_t *link = hash_table_find(&tree->loc_functions, &service_id); if (link != NULL) { - fun = hash_table_get_instance(link, fun_node_t, loc_fun); + fun = hash_table_get_inst(link, fun_node_t, loc_fun); fun_add_ref(fun); } @@ -1499,6 +1510,5 @@ assert(fibril_rwlock_is_write_locked(&tree->rwlock)); - unsigned long key = (unsigned long) fun->service_id; - hash_table_insert(&tree->loc_functions, &key, &fun->loc_fun); + hash_table_insert(&tree->loc_functions, &fun->loc_fun); } Index: uspace/srv/devman/devman.h =================================================================== --- uspace/srv/devman/devman.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/devman/devman.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -52,5 +52,4 @@ #define MATCH_EXT ".ma" -#define DEVICE_BUCKETS 256 #define LOC_DEVICE_NAMESPACE "devices" @@ -151,5 +150,5 @@ * Used by the hash table of devices indexed by devman device handles. */ - link_t devman_dev; + ht_link_t devman_dev; /** @@ -202,10 +201,10 @@ * Used by the hash table of functions indexed by devman device handles. */ - link_t devman_fun; + ht_link_t devman_fun; /** * Used by the hash table of functions indexed by service IDs. */ - link_t loc_fun; + ht_link_t loc_fun; }; Index: uspace/srv/fs/cdfs/cdfs_ops.c =================================================================== --- uspace/srv/fs/cdfs/cdfs_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/cdfs/cdfs_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -37,6 +37,8 @@ */ +#include "cdfs_ops.h" #include #include +#include #include #include @@ -50,21 +52,9 @@ #include "cdfs.h" #include "cdfs_endian.h" -#include "cdfs_ops.h" /** Standard CD-ROM block size */ #define BLOCK_SIZE 2048 -/** Implicit node cache size - * - * More nodes can be actually cached if the files remain - * opended. - * - */ -#define NODE_CACHE_SIZE 200 - -#define NODES_BUCKETS 256 - -#define NODES_KEY_SRVC 0 -#define NODES_KEY_INDEX 1 +#define NODE_CACHE_SIZE 200 /** All root nodes have index 0 */ @@ -205,5 +195,5 @@ service_id_t service_id; /**< Service ID of block device */ - link_t nh_link; /**< Nodes hash table link */ + ht_link_t nh_link; /**< Nodes hash table link */ cdfs_dentry_type_t type; /**< Dentry type */ @@ -226,31 +216,36 @@ static hash_table_t nodes; -static hash_index_t nodes_hash(unsigned long key[]) -{ - return key[NODES_KEY_INDEX] % NODES_BUCKETS; -} - -static int nodes_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - cdfs_node_t *node = - hash_table_get_instance(item, cdfs_node_t, nh_link); - - switch (keys) { - case 1: - return (node->service_id == key[NODES_KEY_SRVC]); - case 2: - return ((node->service_id == key[NODES_KEY_SRVC]) && - (node->index == key[NODES_KEY_INDEX])); - default: - assert((keys == 1) || (keys == 2)); - } - - return 0; -} - -static void nodes_remove_callback(link_t *item) -{ - cdfs_node_t *node = - hash_table_get_instance(item, cdfs_node_t, nh_link); +/* + * Hash table support functions. + */ + +typedef struct { + service_id_t service_id; + fs_index_t index; +} ht_key_t; + +static size_t nodes_key_hash(void *k) +{ + ht_key_t *key = (ht_key_t*)k; + return hash_combine(key->service_id, key->index); +} + +static size_t nodes_hash(const ht_link_t *item) +{ + cdfs_node_t *node = hash_table_get_inst(item, cdfs_node_t, nh_link); + return hash_combine(node->service_id, node->index); +} + +static bool nodes_key_equal(void *k, const ht_link_t *item) +{ + cdfs_node_t *node = hash_table_get_inst(item, cdfs_node_t, nh_link); + ht_key_t *key = (ht_key_t*)k; + + return key->service_id == node->service_id && key->index == node->index; +} + +static void nodes_remove_callback(ht_link_t *item) +{ + cdfs_node_t *node = hash_table_get_inst(item, cdfs_node_t, nh_link); assert(node->type == CDFS_DIRECTORY); @@ -268,7 +263,9 @@ /** Nodes hash table operations */ -static hash_table_operations_t nodes_ops = { +static hash_table_ops_t nodes_ops = { .hash = nodes_hash, - .compare = nodes_compare, + .key_hash = nodes_key_hash, + .key_equal = nodes_key_equal, + .equal = 0, .remove_callback = nodes_remove_callback }; @@ -277,13 +274,13 @@ fs_index_t index) { - unsigned long key[] = { - [NODES_KEY_SRVC] = service_id, - [NODES_KEY_INDEX] = index + ht_key_t key = { + .index = index, + .service_id = service_id }; - link_t *link = hash_table_find(&nodes, key); + ht_link_t *link = hash_table_find(&nodes, &key); if (link) { cdfs_node_t *node = - hash_table_get_instance(link, cdfs_node_t, nh_link); + hash_table_get_inst(link, cdfs_node_t, nh_link); *rfn = FS_NODE(node); @@ -311,5 +308,4 @@ node->opened = 0; - link_initialize(&node->nh_link); list_initialize(&node->cs_list); } @@ -353,10 +349,5 @@ /* Insert the new node into the nodes hash table. */ - unsigned long key[] = { - [NODES_KEY_SRVC] = node->service_id, - [NODES_KEY_INDEX] = node->index - }; - - hash_table_insert(&nodes, key, &node->nh_link); + hash_table_insert(&nodes, &node->nh_link); *rfn = FS_NODE(node); @@ -508,13 +499,13 @@ static fs_node_t *get_cached_node(service_id_t service_id, fs_index_t index) { - unsigned long key[] = { - [NODES_KEY_SRVC] = service_id, - [NODES_KEY_INDEX] = index + ht_key_t key = { + .index = index, + .service_id = service_id }; - link_t *link = hash_table_find(&nodes, key); + ht_link_t *link = hash_table_find(&nodes, &key); if (link) { cdfs_node_t *node = - hash_table_get_instance(link, cdfs_node_t, nh_link); + hash_table_get_inst(link, cdfs_node_t, nh_link); return FS_NODE(node); } @@ -802,11 +793,19 @@ } +static bool rm_service_id_nodes(ht_link_t *item, void *arg) +{ + service_id_t service_id = *(service_id_t*)arg; + cdfs_node_t *node = hash_table_get_inst(item, cdfs_node_t, nh_link); + + if (node->service_id == service_id) { + hash_table_remove_item(&nodes, &node->nh_link); + } + + return true; +} + static void cdfs_instance_done(service_id_t service_id) { - unsigned long key[] = { - [NODES_KEY_SRVC] = service_id - }; - - hash_table_remove(&nodes, key, 1); + hash_table_apply(&nodes, rm_service_id_nodes, &service_id); block_cache_fini(service_id); block_fini(service_id); @@ -822,15 +821,15 @@ size_t *rbytes) { - unsigned long key[] = { - [NODES_KEY_SRVC] = service_id, - [NODES_KEY_INDEX] = index + ht_key_t key = { + .index = index, + .service_id = service_id }; - link_t *link = hash_table_find(&nodes, key); + ht_link_t *link = hash_table_find(&nodes, &key); if (link == NULL) return ENOENT; cdfs_node_t *node = - hash_table_get_instance(link, cdfs_node_t, nh_link); + hash_table_get_inst(link, cdfs_node_t, nh_link); if (!node->processed) { @@ -912,34 +911,31 @@ } +static bool cache_remove_closed(ht_link_t *item, void *arg) +{ + size_t *premove_cnt = (size_t*)arg; + + /* Some nodes were requested to be removed from the cache. */ + if (0 < *premove_cnt) { + cdfs_node_t *node = hash_table_get_inst(item, cdfs_node_t, nh_link); + + if (!node->opened) { + hash_table_remove_item(&nodes, item); + + --nodes_cached; + --*premove_cnt; + } + } + + /* Only continue if more nodes were requested to be removed. */ + return 0 < *premove_cnt; +} + static void cleanup_cache(service_id_t service_id) { if (nodes_cached > NODE_CACHE_SIZE) { - size_t remove = nodes_cached - NODE_CACHE_SIZE; - - // FIXME: this accesses the internals of the hash table - // and should be rewritten in a clean way - - for (hash_index_t chain = 0; chain < nodes.entries; chain++) { - for (link_t *link = nodes.entry[chain].head.next; - link != &nodes.entry[chain].head; - link = link->next) { - if (remove == 0) - return; - - cdfs_node_t *node = - hash_table_get_instance(link, cdfs_node_t, nh_link); - if (node->opened == 0) { - link_t *tmp = link; - link = link->prev; - - list_remove(tmp); - nodes.op->remove_callback(tmp); - nodes_cached--; - remove--; - - continue; - } - } - } + size_t remove_cnt = nodes_cached - NODE_CACHE_SIZE; + + if (0 < remove_cnt) + hash_table_apply(&nodes, cache_remove_closed, &remove_cnt); } } @@ -951,15 +947,15 @@ return EOK; - unsigned long key[] = { - [NODES_KEY_SRVC] = service_id, - [NODES_KEY_INDEX] = index + ht_key_t key = { + .index = index, + .service_id = service_id }; - link_t *link = hash_table_find(&nodes, key); + ht_link_t *link = hash_table_find(&nodes, &key); if (link == 0) return ENOENT; cdfs_node_t *node = - hash_table_get_instance(link, cdfs_node_t, nh_link); + hash_table_get_inst(link, cdfs_node_t, nh_link); assert(node->opened > 0); @@ -1007,5 +1003,5 @@ bool cdfs_init(void) { - if (!hash_table_create(&nodes, NODES_BUCKETS, 2, &nodes_ops)) + if (!hash_table_create(&nodes, 0, 0, &nodes_ops)) return false; Index: uspace/srv/fs/exfat/exfat.h =================================================================== --- uspace/srv/fs/exfat/exfat.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/exfat/exfat.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -106,7 +106,7 @@ typedef struct { /** Used indices (position) hash table link. */ - link_t uph_link; + ht_link_t uph_link; /** Used indices (index) hash table link. */ - link_t uih_link; + ht_link_t uih_link; fibril_mutex_t lock; Index: uspace/srv/fs/exfat/exfat_idx.c =================================================================== --- uspace/srv/fs/exfat/exfat_idx.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/exfat/exfat_idx.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include #include #include @@ -91,4 +92,5 @@ if (lock) fibril_mutex_lock(&unused_lock); + list_foreach(unused_list, l) { u = list_get_instance(l, unused_t, link); @@ -112,70 +114,48 @@ static hash_table_t up_hash; -#define UPH_BUCKETS_LOG 12 -#define UPH_BUCKETS (1 << UPH_BUCKETS_LOG) - -#define UPH_SID_KEY 0 -#define UPH_PFC_KEY 1 -#define UPH_PDI_KEY 2 - -static hash_index_t pos_hash(unsigned long key[]) -{ - service_id_t service_id = (service_id_t)key[UPH_SID_KEY]; - exfat_cluster_t pfc = (exfat_cluster_t)key[UPH_PFC_KEY]; - unsigned pdi = (unsigned)key[UPH_PDI_KEY]; - - hash_index_t h; - - /* - * The least significant half of all bits are the least significant bits - * of the parent node's first cluster. - * - * The least significant half of the most significant half of all bits - * are the least significant bits of the node's dentry index within the - * parent directory node. - * - * The most significant half of the most significant half of all bits - * are the least significant bits of the device handle. - */ - h = pfc & ((1 << (UPH_BUCKETS_LOG / 2)) - 1); - h |= (pdi & ((1 << (UPH_BUCKETS_LOG / 4)) - 1)) << - (UPH_BUCKETS_LOG / 2); - h |= (service_id & ((1 << (UPH_BUCKETS_LOG / 4)) - 1)) << - (3 * (UPH_BUCKETS_LOG / 4)); - - return h; -} - -static int pos_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - service_id_t service_id = (service_id_t)key[UPH_SID_KEY]; +typedef struct { + service_id_t service_id; exfat_cluster_t pfc; unsigned pdi; - exfat_idx_t *fidx = list_get_instance(item, exfat_idx_t, uph_link); - - switch (keys) { - case 1: - return (service_id == fidx->service_id); - case 3: - pfc = (exfat_cluster_t) key[UPH_PFC_KEY]; - pdi = (unsigned) key[UPH_PDI_KEY]; - return (service_id == fidx->service_id) && (pfc == fidx->pfc) && - (pdi == fidx->pdi); - default: - assert((keys == 1) || (keys == 3)); - } - - return 0; -} - -static void pos_remove_callback(link_t *item) -{ - /* nothing to do */ -} - -static hash_table_operations_t uph_ops = { +} pos_key_t; + +static inline size_t pos_key_hash(void *key) +{ + pos_key_t *pos = (pos_key_t*)key; + + size_t hash = 0; + hash = hash_combine(pos->pfc, pos->pdi); + return hash_combine(hash, pos->service_id); +} + +static size_t pos_hash(const ht_link_t *item) +{ + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uph_link); + + pos_key_t pkey = { + .service_id = fidx->service_id, + .pfc = fidx->pfc, + .pdi = fidx->pdi, + }; + + return pos_key_hash(&pkey); +} + +static bool pos_key_equal(void *key, const ht_link_t *item) +{ + pos_key_t *pos = (pos_key_t*)key; + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uph_link); + + return pos->service_id == fidx->service_id + && pos->pdi == fidx->pdi + && pos->pfc == fidx->pfc; +} + +static hash_table_ops_t uph_ops = { .hash = pos_hash, - .compare = pos_compare, - .remove_callback = pos_remove_callback, + .key_hash = pos_key_hash, + .key_equal = pos_key_equal, + .equal = 0, + .remove_callback = 0, }; @@ -186,54 +166,41 @@ static hash_table_t ui_hash; -#define UIH_BUCKETS_LOG 12 -#define UIH_BUCKETS (1 << UIH_BUCKETS_LOG) - -#define UIH_SID_KEY 0 -#define UIH_INDEX_KEY 1 - -static hash_index_t idx_hash(unsigned long key[]) -{ - service_id_t service_id = (service_id_t)key[UIH_SID_KEY]; - fs_index_t index = (fs_index_t)key[UIH_INDEX_KEY]; - - hash_index_t h; - - h = service_id & ((1 << (UIH_BUCKETS_LOG / 2)) - 1); - h |= (index & ((1 << (UIH_BUCKETS_LOG / 2)) - 1)) << - (UIH_BUCKETS_LOG / 2); - - return h; -} - -static int idx_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - service_id_t service_id = (service_id_t)key[UIH_SID_KEY]; +typedef struct { + service_id_t service_id; fs_index_t index; - exfat_idx_t *fidx = list_get_instance(item, exfat_idx_t, uih_link); - - switch (keys) { - case 1: - return (service_id == fidx->service_id); - case 2: - index = (fs_index_t) key[UIH_INDEX_KEY]; - return (service_id == fidx->service_id) && - (index == fidx->index); - default: - assert((keys == 1) || (keys == 2)); - } - - return 0; -} - -static void idx_remove_callback(link_t *item) -{ - exfat_idx_t *fidx = list_get_instance(item, exfat_idx_t, uih_link); +} idx_key_t; + +static size_t idx_key_hash(void *key_arg) +{ + idx_key_t *key = (idx_key_t*)key_arg; + return hash_combine(key->service_id, key->index); +} + +static size_t idx_hash(const ht_link_t *item) +{ + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uih_link); + return hash_combine(fidx->service_id, fidx->index); +} + +static bool idx_key_equal(void *key_arg, const ht_link_t *item) +{ + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uih_link); + idx_key_t *key = (idx_key_t*)key_arg; + + return key->index == fidx->index && key->service_id == fidx->service_id; +} + +static void idx_remove_callback(ht_link_t *item) +{ + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uih_link); free(fidx); } -static hash_table_operations_t uih_ops = { +static hash_table_ops_t uih_ops = { .hash = idx_hash, - .compare = idx_compare, + .key_hash = idx_key_hash, + .key_equal = idx_key_equal, + .equal = 0, .remove_callback = idx_remove_callback, }; @@ -376,6 +343,4 @@ } - link_initialize(&fidx->uph_link); - link_initialize(&fidx->uih_link); fibril_mutex_initialize(&fidx->lock); fidx->service_id = service_id; @@ -400,10 +365,5 @@ } - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = fidx->index, - }; - - hash_table_insert(&ui_hash, ikey, &fidx->uih_link); + hash_table_insert(&ui_hash, &fidx->uih_link); fibril_mutex_lock(&fidx->lock); fibril_mutex_unlock(&used_lock); @@ -417,15 +377,15 @@ { exfat_idx_t *fidx; - link_t *l; - unsigned long pkey[] = { - [UPH_SID_KEY] = service_id, - [UPH_PFC_KEY] = pfc, - [UPH_PDI_KEY] = pdi, + + pos_key_t pos_key = { + .service_id = service_id, + .pfc = pfc, + .pdi = pdi, }; fibril_mutex_lock(&used_lock); - l = hash_table_find(&up_hash, pkey); + ht_link_t *l = hash_table_find(&up_hash, &pos_key); if (l) { - fidx = hash_table_get_instance(l, exfat_idx_t, uph_link); + fidx = hash_table_get_inst(l, exfat_idx_t, uph_link); } else { int rc; @@ -437,14 +397,9 @@ } - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = fidx->index, - }; - fidx->pfc = pfc; fidx->pdi = pdi; - hash_table_insert(&up_hash, pkey, &fidx->uph_link); - hash_table_insert(&ui_hash, ikey, &fidx->uih_link); + hash_table_insert(&up_hash, &fidx->uph_link); + hash_table_insert(&ui_hash, &fidx->uih_link); } fibril_mutex_lock(&fidx->lock); @@ -456,12 +411,6 @@ void exfat_idx_hashin(exfat_idx_t *idx) { - unsigned long pkey[] = { - [UPH_SID_KEY] = idx->service_id, - [UPH_PFC_KEY] = idx->pfc, - [UPH_PDI_KEY] = idx->pdi, - }; - - fibril_mutex_lock(&used_lock); - hash_table_insert(&up_hash, pkey, &idx->uph_link); + fibril_mutex_lock(&used_lock); + hash_table_insert(&up_hash, &idx->uph_link); fibril_mutex_unlock(&used_lock); } @@ -469,12 +418,6 @@ void exfat_idx_hashout(exfat_idx_t *idx) { - unsigned long pkey[] = { - [UPH_SID_KEY] = idx->service_id, - [UPH_PFC_KEY] = idx->pfc, - [UPH_PDI_KEY] = idx->pdi, - }; - - fibril_mutex_lock(&used_lock); - hash_table_remove(&up_hash, pkey, 3); + fibril_mutex_lock(&used_lock); + hash_table_remove_item(&up_hash, &idx->uph_link); fibril_mutex_unlock(&used_lock); } @@ -484,14 +427,14 @@ { exfat_idx_t *fidx = NULL; - link_t *l; - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = index, + + idx_key_t idx_key = { + .service_id = service_id, + .index = index, }; fibril_mutex_lock(&used_lock); - l = hash_table_find(&ui_hash, ikey); + ht_link_t *l = hash_table_find(&ui_hash, &idx_key); if (l) { - fidx = hash_table_get_instance(l, exfat_idx_t, uih_link); + fidx = hash_table_get_inst(l, exfat_idx_t, uih_link); fibril_mutex_lock(&fidx->lock); } @@ -507,10 +450,8 @@ void exfat_idx_destroy(exfat_idx_t *idx) { - unsigned long ikey[] = { - [UIH_SID_KEY] = idx->service_id, - [UIH_INDEX_KEY] = idx->index, + idx_key_t idx_key = { + .service_id = idx->service_id, + .index = idx->index, }; - service_id_t service_id = idx->service_id; - fs_index_t index = idx->index; /* TODO: assert(idx->pfc == FAT_CLST_RES0); */ @@ -523,8 +464,8 @@ * the index hash only. */ - hash_table_remove(&ui_hash, ikey, 2); + hash_table_remove(&ui_hash, &idx_key); fibril_mutex_unlock(&used_lock); /* Release the VFS index. */ - exfat_index_free(service_id, index); + exfat_index_free(idx_key.service_id, idx_key.index); /* The index structure itself is freed in idx_remove_callback(). */ } @@ -532,7 +473,7 @@ int exfat_idx_init(void) { - if (!hash_table_create(&up_hash, UPH_BUCKETS, 3, &uph_ops)) + if (!hash_table_create(&up_hash, 0, 0, &uph_ops)) return ENOMEM; - if (!hash_table_create(&ui_hash, UIH_BUCKETS, 2, &uih_ops)) { + if (!hash_table_create(&ui_hash, 0, 0, &uih_ops)) { hash_table_destroy(&up_hash); return ENOMEM; @@ -544,4 +485,5 @@ { /* We assume the hash tables are empty. */ + assert(hash_table_empty(&up_hash) && hash_table_empty(&ui_hash)); hash_table_destroy(&up_hash); hash_table_destroy(&ui_hash); @@ -568,13 +510,30 @@ } +static bool rm_pos_service_id(ht_link_t *item, void *arg) +{ + service_id_t service_id = *(service_id_t*)arg; + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uph_link); + + if (fidx->service_id == service_id) { + hash_table_remove_item(&up_hash, item); + } + + return true; +} + +static bool rm_idx_service_id(ht_link_t *item, void *arg) +{ + service_id_t service_id = *(service_id_t*)arg; + exfat_idx_t *fidx = hash_table_get_inst(item, exfat_idx_t, uih_link); + + if (fidx->service_id == service_id) { + hash_table_remove_item(&ui_hash, item); + } + + return true; +} + void exfat_idx_fini_by_service_id(service_id_t service_id) { - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id - }; - unsigned long pkey[] = { - [UPH_SID_KEY] = service_id - }; - /* * Remove this instance's index structure from up_hash and ui_hash. @@ -583,6 +542,6 @@ */ fibril_mutex_lock(&used_lock); - hash_table_remove(&up_hash, pkey, 1); - hash_table_remove(&ui_hash, ikey, 1); + hash_table_apply(&up_hash, rm_pos_service_id, &service_id); + hash_table_apply(&ui_hash, rm_idx_service_id, &service_id); fibril_mutex_unlock(&used_lock); Index: uspace/srv/fs/exfat/exfat_ops.c =================================================================== --- uspace/srv/fs/exfat/exfat_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/exfat/exfat_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -54,4 +54,5 @@ #include #include +#include #include #include Index: uspace/srv/fs/ext2fs/ext2fs_ops.c =================================================================== --- uspace/srv/fs/ext2fs/ext2fs_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/ext2fs/ext2fs_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -49,4 +49,5 @@ #include #include +#include #include #include @@ -62,8 +63,4 @@ #define EXT2FS_NODE(node) ((node) ? (ext2fs_node_t *) (node)->data : NULL) #define EXT2FS_DBG(format, ...) {if (false) printf("ext2fs: %s: " format "\n", __FUNCTION__, ##__VA_ARGS__);} -#define OPEN_NODES_KEYS 2 -#define OPEN_NODES_DEV_HANDLE_KEY 0 -#define OPEN_NODES_INODE_KEY 1 -#define OPEN_NODES_BUCKETS 256 typedef struct ext2fs_instance { @@ -78,5 +75,5 @@ ext2_inode_ref_t *inode_ref; fs_node_t *fs_node; - link_t link; + ht_link_t link; unsigned int references; } ext2fs_node_t; @@ -122,36 +119,44 @@ static FIBRIL_MUTEX_INITIALIZE(open_nodes_lock); -/* Hash table interface for open nodes hash table */ -static hash_index_t open_nodes_hash(unsigned long key[]) -{ - /* TODO: This is very simple and probably can be improved */ - return key[OPEN_NODES_INODE_KEY] % OPEN_NODES_BUCKETS; -} - -static int open_nodes_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - ext2fs_node_t *enode = hash_table_get_instance(item, ext2fs_node_t, link); - assert(keys > 0); - if (enode->instance->service_id != - ((service_id_t) key[OPEN_NODES_DEV_HANDLE_KEY])) { - return false; - } - if (keys == 1) { - return true; - } - assert(keys == 2); - return (enode->inode_ref->index == key[OPEN_NODES_INODE_KEY]); -} - -static void open_nodes_remove_cb(link_t *link) -{ - /* We don't use remove callback for this hash table */ -} - -static hash_table_operations_t open_nodes_ops = { +/* + * Hash table interface for open nodes hash table + */ + +typedef struct { + service_id_t service_id; + fs_index_t index; +} node_key_t; + +static size_t open_nodes_key_hash(void *key) +{ + node_key_t *node_key = (node_key_t*)key; + return hash_combine(node_key->service_id, node_key->index); +} + +static size_t open_nodes_hash(const ht_link_t *item) +{ + ext2fs_node_t *enode = hash_table_get_inst(item, ext2fs_node_t, link); + + assert(enode->instance); + assert(enode->inode_ref); + + return hash_combine(enode->instance->service_id, enode->inode_ref->index); +} + +static bool open_nodes_key_equal(void *key, const ht_link_t *item) +{ + node_key_t *node_key = (node_key_t*)key; + ext2fs_node_t *enode = hash_table_get_inst(item, ext2fs_node_t, link); + + return node_key->service_id == enode->instance->service_id + && node_key->index == enode->inode_ref->index; +} + +static hash_table_ops_t open_nodes_ops = { .hash = open_nodes_hash, - .compare = open_nodes_compare, - .remove_callback = open_nodes_remove_cb, + .key_hash = open_nodes_key_hash, + .key_equal = open_nodes_key_equal, + .equal = 0, + .remove_callback = 0, }; @@ -161,6 +166,5 @@ int ext2fs_global_init(void) { - if (!hash_table_create(&open_nodes, OPEN_NODES_BUCKETS, - OPEN_NODES_KEYS, &open_nodes_ops)) { + if (!hash_table_create(&open_nodes, 0, 0, &open_nodes_ops)) { return ENOMEM; } @@ -316,12 +320,12 @@ /* Check if the node is not already open */ - unsigned long key[] = { - [OPEN_NODES_DEV_HANDLE_KEY] = inst->service_id, - [OPEN_NODES_INODE_KEY] = index, + node_key_t key = { + .service_id = inst->service_id, + .index = index }; - link_t *already_open = hash_table_find(&open_nodes, key); + ht_link_t *already_open = hash_table_find(&open_nodes, &key); if (already_open) { - enode = hash_table_get_instance(already_open, ext2fs_node_t, link); + enode = hash_table_get_inst(already_open, ext2fs_node_t, link); *rfn = enode->fs_node; enode->references++; @@ -357,10 +361,9 @@ enode->references = 1; enode->fs_node = node; - link_initialize(&enode->link); node->data = enode; *rfn = node; - hash_table_insert(&open_nodes, key, &enode->link); + hash_table_insert(&open_nodes, &enode->link); inst->open_nodes_count++; @@ -408,15 +411,15 @@ int ext2fs_node_put_core(ext2fs_node_t *enode) { - int rc; - - unsigned long key[] = { - [OPEN_NODES_DEV_HANDLE_KEY] = enode->instance->service_id, - [OPEN_NODES_INODE_KEY] = enode->inode_ref->index, + node_key_t key = { + .service_id = enode->instance->service_id, + .index = enode->inode_ref->index }; - hash_table_remove(&open_nodes, key, OPEN_NODES_KEYS); + + hash_table_remove(&open_nodes, &key); + assert(enode->instance->open_nodes_count > 0); enode->instance->open_nodes_count--; - rc = ext2_filesystem_put_inode_ref(enode->inode_ref); + int rc = ext2_filesystem_put_inode_ref(enode->inode_ref); if (rc != EOK) { EXT2FS_DBG("ext2_filesystem_put_inode_ref failed"); Index: uspace/srv/fs/fat/fat.h =================================================================== --- uspace/srv/fs/fat/fat.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/fat/fat.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -190,7 +190,7 @@ typedef struct { /** Used indices (position) hash table link. */ - link_t uph_link; + ht_link_t uph_link; /** Used indices (index) hash table link. */ - link_t uih_link; + ht_link_t uih_link; fibril_mutex_t lock; Index: uspace/srv/fs/fat/fat_idx.c =================================================================== --- uspace/srv/fs/fat/fat_idx.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/fat/fat_idx.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include #include #include @@ -58,6 +59,6 @@ */ typedef struct { - link_t link; - service_id_t service_id; + link_t link; + service_id_t service_id; /** Next unassigned index. */ @@ -97,5 +98,5 @@ return u; } - + if (lock) fibril_mutex_unlock(&unused_lock); @@ -113,70 +114,48 @@ static hash_table_t up_hash; -#define UPH_BUCKETS_LOG 12 -#define UPH_BUCKETS (1 << UPH_BUCKETS_LOG) - -#define UPH_SID_KEY 0 -#define UPH_PFC_KEY 1 -#define UPH_PDI_KEY 2 - -static hash_index_t pos_hash(unsigned long key[]) -{ - service_id_t service_id = (service_id_t)key[UPH_SID_KEY]; - fat_cluster_t pfc = (fat_cluster_t)key[UPH_PFC_KEY]; - unsigned pdi = (unsigned)key[UPH_PDI_KEY]; - - hash_index_t h; - - /* - * The least significant half of all bits are the least significant bits - * of the parent node's first cluster. - * - * The least significant half of the most significant half of all bits - * are the least significant bits of the node's dentry index within the - * parent directory node. - * - * The most significant half of the most significant half of all bits - * are the least significant bits of the device handle. - */ - h = pfc & ((1 << (UPH_BUCKETS_LOG / 2)) - 1); - h |= (pdi & ((1 << (UPH_BUCKETS_LOG / 4)) - 1)) << - (UPH_BUCKETS_LOG / 2); - h |= (service_id & ((1 << (UPH_BUCKETS_LOG / 4)) - 1)) << - (3 * (UPH_BUCKETS_LOG / 4)); - - return h; -} - -static int pos_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - service_id_t service_id = (service_id_t)key[UPH_SID_KEY]; +typedef struct { + service_id_t service_id; fat_cluster_t pfc; unsigned pdi; - fat_idx_t *fidx = list_get_instance(item, fat_idx_t, uph_link); - - switch (keys) { - case 1: - return (service_id == fidx->service_id); - case 3: - pfc = (fat_cluster_t) key[UPH_PFC_KEY]; - pdi = (unsigned) key[UPH_PDI_KEY]; - return (service_id == fidx->service_id) && (pfc == fidx->pfc) && - (pdi == fidx->pdi); - default: - assert((keys == 1) || (keys == 3)); - } - - return 0; -} - -static void pos_remove_callback(link_t *item) -{ - /* nothing to do */ -} - -static hash_table_operations_t uph_ops = { +} pos_key_t; + +static inline size_t pos_key_hash(void *key) +{ + pos_key_t *pos = (pos_key_t*)key; + + size_t hash = 0; + hash = hash_combine(pos->pfc, pos->pdi); + return hash_combine(hash, pos->service_id); +} + +static size_t pos_hash(const ht_link_t *item) +{ + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uph_link); + + pos_key_t pkey = { + .service_id = fidx->service_id, + .pfc = fidx->pfc, + .pdi = fidx->pdi, + }; + + return pos_key_hash(&pkey); +} + +static bool pos_key_equal(void *key, const ht_link_t *item) +{ + pos_key_t *pos = (pos_key_t*)key; + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uph_link); + + return pos->service_id == fidx->service_id + && pos->pdi == fidx->pdi + && pos->pfc == fidx->pfc; +} + +static hash_table_ops_t uph_ops = { .hash = pos_hash, - .compare = pos_compare, - .remove_callback = pos_remove_callback, + .key_hash = pos_key_hash, + .key_equal = pos_key_equal, + .equal = 0, + .remove_callback = 0, }; @@ -187,54 +166,41 @@ static hash_table_t ui_hash; -#define UIH_BUCKETS_LOG 12 -#define UIH_BUCKETS (1 << UIH_BUCKETS_LOG) - -#define UIH_SID_KEY 0 -#define UIH_INDEX_KEY 1 - -static hash_index_t idx_hash(unsigned long key[]) -{ - service_id_t service_id = (service_id_t)key[UIH_SID_KEY]; - fs_index_t index = (fs_index_t)key[UIH_INDEX_KEY]; - - hash_index_t h; - - h = service_id & ((1 << (UIH_BUCKETS_LOG / 2)) - 1); - h |= (index & ((1 << (UIH_BUCKETS_LOG / 2)) - 1)) << - (UIH_BUCKETS_LOG / 2); - - return h; -} - -static int idx_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - service_id_t service_id = (service_id_t)key[UIH_SID_KEY]; +typedef struct { + service_id_t service_id; fs_index_t index; - fat_idx_t *fidx = list_get_instance(item, fat_idx_t, uih_link); - - switch (keys) { - case 1: - return (service_id == fidx->service_id); - case 2: - index = (fs_index_t) key[UIH_INDEX_KEY]; - return (service_id == fidx->service_id) && - (index == fidx->index); - default: - assert((keys == 1) || (keys == 2)); - } - - return 0; -} - -static void idx_remove_callback(link_t *item) -{ - fat_idx_t *fidx = list_get_instance(item, fat_idx_t, uih_link); +} idx_key_t; + +static size_t idx_key_hash(void *key_arg) +{ + idx_key_t *key = (idx_key_t*)key_arg; + return hash_combine(key->service_id, key->index); +} + +static size_t idx_hash(const ht_link_t *item) +{ + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uih_link); + return hash_combine(fidx->service_id, fidx->index); +} + +static bool idx_key_equal(void *key_arg, const ht_link_t *item) +{ + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uih_link); + idx_key_t *key = (idx_key_t*)key_arg; + + return key->index == fidx->index && key->service_id == fidx->service_id; +} + +static void idx_remove_callback(ht_link_t *item) +{ + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uih_link); free(fidx); } -static hash_table_operations_t uih_ops = { +static hash_table_ops_t uih_ops = { .hash = idx_hash, - .compare = idx_compare, + .key_hash = idx_key_hash, + .key_equal = idx_key_equal, + .equal = 0, .remove_callback = idx_remove_callback, }; @@ -377,6 +343,4 @@ } - link_initialize(&fidx->uph_link); - link_initialize(&fidx->uih_link); fibril_mutex_initialize(&fidx->lock); fidx->service_id = service_id; @@ -401,10 +365,5 @@ } - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = fidx->index, - }; - - hash_table_insert(&ui_hash, ikey, &fidx->uih_link); + hash_table_insert(&ui_hash, &fidx->uih_link); fibril_mutex_lock(&fidx->lock); fibril_mutex_unlock(&used_lock); @@ -418,15 +377,15 @@ { fat_idx_t *fidx; - link_t *l; - unsigned long pkey[] = { - [UPH_SID_KEY] = service_id, - [UPH_PFC_KEY] = pfc, - [UPH_PDI_KEY] = pdi, + + pos_key_t pos_key = { + .service_id = service_id, + .pfc = pfc, + .pdi = pdi, }; fibril_mutex_lock(&used_lock); - l = hash_table_find(&up_hash, pkey); + ht_link_t *l = hash_table_find(&up_hash, &pos_key); if (l) { - fidx = hash_table_get_instance(l, fat_idx_t, uph_link); + fidx = hash_table_get_inst(l, fat_idx_t, uph_link); } else { int rc; @@ -438,14 +397,9 @@ } - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = fidx->index, - }; - fidx->pfc = pfc; fidx->pdi = pdi; - hash_table_insert(&up_hash, pkey, &fidx->uph_link); - hash_table_insert(&ui_hash, ikey, &fidx->uih_link); + hash_table_insert(&up_hash, &fidx->uph_link); + hash_table_insert(&ui_hash, &fidx->uih_link); } fibril_mutex_lock(&fidx->lock); @@ -457,12 +411,6 @@ void fat_idx_hashin(fat_idx_t *idx) { - unsigned long pkey[] = { - [UPH_SID_KEY] = idx->service_id, - [UPH_PFC_KEY] = idx->pfc, - [UPH_PDI_KEY] = idx->pdi, - }; - - fibril_mutex_lock(&used_lock); - hash_table_insert(&up_hash, pkey, &idx->uph_link); + fibril_mutex_lock(&used_lock); + hash_table_insert(&up_hash, &idx->uph_link); fibril_mutex_unlock(&used_lock); } @@ -470,12 +418,6 @@ void fat_idx_hashout(fat_idx_t *idx) { - unsigned long pkey[] = { - [UPH_SID_KEY] = idx->service_id, - [UPH_PFC_KEY] = idx->pfc, - [UPH_PDI_KEY] = idx->pdi, - }; - - fibril_mutex_lock(&used_lock); - hash_table_remove(&up_hash, pkey, 3); + fibril_mutex_lock(&used_lock); + hash_table_remove_item(&up_hash, &idx->uph_link); fibril_mutex_unlock(&used_lock); } @@ -485,14 +427,14 @@ { fat_idx_t *fidx = NULL; - link_t *l; - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id, - [UIH_INDEX_KEY] = index, + + idx_key_t idx_key = { + .service_id = service_id, + .index = index, }; fibril_mutex_lock(&used_lock); - l = hash_table_find(&ui_hash, ikey); + ht_link_t *l = hash_table_find(&ui_hash, &idx_key); if (l) { - fidx = hash_table_get_instance(l, fat_idx_t, uih_link); + fidx = hash_table_get_inst(l, fat_idx_t, uih_link); fibril_mutex_lock(&fidx->lock); } @@ -508,10 +450,8 @@ void fat_idx_destroy(fat_idx_t *idx) { - unsigned long ikey[] = { - [UIH_SID_KEY] = idx->service_id, - [UIH_INDEX_KEY] = idx->index, + idx_key_t idx_key = { + .service_id = idx->service_id, + .index = idx->index, }; - service_id_t service_id = idx->service_id; - fs_index_t index = idx->index; assert(idx->pfc == FAT_CLST_RES0); @@ -523,8 +463,8 @@ * the index hash only. */ - hash_table_remove(&ui_hash, ikey, 2); + hash_table_remove(&ui_hash, &idx_key); fibril_mutex_unlock(&used_lock); /* Release the VFS index. */ - fat_index_free(service_id, index); + fat_index_free(idx_key.service_id, idx_key.index); /* The index structure itself is freed in idx_remove_callback(). */ } @@ -532,7 +472,7 @@ int fat_idx_init(void) { - if (!hash_table_create(&up_hash, UPH_BUCKETS, 3, &uph_ops)) + if (!hash_table_create(&up_hash, 0, 0, &uph_ops)) return ENOMEM; - if (!hash_table_create(&ui_hash, UIH_BUCKETS, 2, &uih_ops)) { + if (!hash_table_create(&ui_hash, 0, 0, &uih_ops)) { hash_table_destroy(&up_hash); return ENOMEM; @@ -544,4 +484,5 @@ { /* We assume the hash tables are empty. */ + assert(hash_table_empty(&up_hash) && hash_table_empty(&ui_hash)); hash_table_destroy(&up_hash); hash_table_destroy(&ui_hash); @@ -568,13 +509,30 @@ } +static bool rm_pos_service_id(ht_link_t *item, void *arg) +{ + service_id_t service_id = *(service_id_t*)arg; + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uph_link); + + if (fidx->service_id == service_id) { + hash_table_remove_item(&up_hash, item); + } + + return true; +} + +static bool rm_idx_service_id(ht_link_t *item, void *arg) +{ + service_id_t service_id = *(service_id_t*)arg; + fat_idx_t *fidx = hash_table_get_inst(item, fat_idx_t, uih_link); + + if (fidx->service_id == service_id) { + hash_table_remove_item(&ui_hash, item); + } + + return true; +} + void fat_idx_fini_by_service_id(service_id_t service_id) { - unsigned long ikey[] = { - [UIH_SID_KEY] = service_id - }; - unsigned long pkey[] = { - [UPH_SID_KEY] = service_id - }; - /* * Remove this instance's index structure from up_hash and ui_hash. @@ -583,6 +541,6 @@ */ fibril_mutex_lock(&used_lock); - hash_table_remove(&up_hash, pkey, 1); - hash_table_remove(&ui_hash, ikey, 1); + hash_table_apply(&up_hash, rm_pos_service_id, &service_id); + hash_table_apply(&ui_hash, rm_idx_service_id, &service_id); fibril_mutex_unlock(&used_lock); Index: uspace/srv/fs/locfs/locfs_ops.c =================================================================== --- uspace/srv/fs/locfs/locfs_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/locfs/locfs_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -61,5 +61,5 @@ async_sess_t *sess; /**< If NULL, the structure is incomplete. */ size_t refcount; - link_t link; + ht_link_t link; fibril_condvar_t cv; /**< Broadcast when completed. */ } service_t; @@ -71,28 +71,33 @@ static FIBRIL_MUTEX_INITIALIZE(services_mutex); -#define SERVICES_KEYS 1 -#define SERVICES_KEY_HANDLE 0 -#define SERVICES_BUCKETS 256 - /* Implementation of hash table interface for the nodes hash table. */ -static hash_index_t services_hash(unsigned long key[]) -{ - return key[SERVICES_KEY_HANDLE] % SERVICES_BUCKETS; -} - -static int services_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - service_t *dev = hash_table_get_instance(item, service_t, link); - return (dev->service_id == (service_id_t) key[SERVICES_KEY_HANDLE]); -} - -static void services_remove_callback(link_t *item) -{ - free(hash_table_get_instance(item, service_t, link)); -} - -static hash_table_operations_t services_ops = { + +static size_t services_key_hash(void *key) +{ + return *(service_id_t*)key; +} + +static size_t services_hash(const ht_link_t *item) +{ + service_t *dev = hash_table_get_inst(item, service_t, link); + return dev->service_id; +} + +static bool services_key_equal(void *key, const ht_link_t *item) +{ + service_t *dev = hash_table_get_inst(item, service_t, link); + return (dev->service_id == *(service_id_t*)key); +} + +static void services_remove_callback(ht_link_t *item) +{ + free(hash_table_get_inst(item, service_t, link)); +} + +static hash_table_ops_t services_ops = { .hash = services_hash, - .compare = services_compare, + .key_hash = services_key_hash, + .key_equal = services_key_equal, + .equal = 0, .remove_callback = services_remove_callback }; @@ -229,12 +234,8 @@ /* Device node */ - unsigned long key[] = { - [SERVICES_KEY_HANDLE] = (unsigned long) node->service_id - }; - link_t *lnk; - fibril_mutex_lock(&services_mutex); + ht_link_t *lnk; restart: - lnk = hash_table_find(&services, key); + lnk = hash_table_find(&services, &node->service_id); if (lnk == NULL) { service_t *dev = (service_t *) malloc(sizeof(service_t)); @@ -256,5 +257,5 @@ * below. */ - hash_table_insert(&services, key, &dev->link); + hash_table_insert(&services, &dev->link); /* @@ -279,5 +280,5 @@ * entry and free the device structure. */ - hash_table_remove(&services, key, SERVICES_KEYS); + hash_table_remove(&services, &node->service_id); fibril_mutex_unlock(&services_mutex); @@ -288,5 +289,5 @@ dev->sess = sess; } else { - service_t *dev = hash_table_get_instance(lnk, service_t, link); + service_t *dev = hash_table_get_inst(lnk, service_t, link); if (!dev->sess) { @@ -450,6 +451,5 @@ bool locfs_init(void) { - if (!hash_table_create(&services, SERVICES_BUCKETS, - SERVICES_KEYS, &services_ops)) + if (!hash_table_create(&services, 0, 0, &services_ops)) return false; @@ -555,10 +555,6 @@ /* Device node */ - unsigned long key[] = { - [SERVICES_KEY_HANDLE] = (unsigned long) index - }; - fibril_mutex_lock(&services_mutex); - link_t *lnk = hash_table_find(&services, key); + ht_link_t *lnk = hash_table_find(&services, &index); if (lnk == NULL) { fibril_mutex_unlock(&services_mutex); @@ -566,5 +562,5 @@ } - service_t *dev = hash_table_get_instance(lnk, service_t, link); + service_t *dev = hash_table_get_inst(lnk, service_t, link); assert(dev->sess); @@ -621,10 +617,7 @@ if (type == LOC_OBJECT_SERVICE) { /* Device node */ - unsigned long key[] = { - [SERVICES_KEY_HANDLE] = (unsigned long) index - }; fibril_mutex_lock(&services_mutex); - link_t *lnk = hash_table_find(&services, key); + ht_link_t *lnk = hash_table_find(&services, &index); if (lnk == NULL) { fibril_mutex_unlock(&services_mutex); @@ -632,5 +625,5 @@ } - service_t *dev = hash_table_get_instance(lnk, service_t, link); + service_t *dev = hash_table_get_inst(lnk, service_t, link); assert(dev->sess); @@ -691,10 +684,7 @@ if (type == LOC_OBJECT_SERVICE) { - unsigned long key[] = { - [SERVICES_KEY_HANDLE] = (unsigned long) index - }; fibril_mutex_lock(&services_mutex); - link_t *lnk = hash_table_find(&services, key); + ht_link_t *lnk = hash_table_find(&services, &index); if (lnk == NULL) { fibril_mutex_unlock(&services_mutex); @@ -702,5 +692,5 @@ } - service_t *dev = hash_table_get_instance(lnk, service_t, link); + service_t *dev = hash_table_get_inst(lnk, service_t, link); assert(dev->sess); dev->refcount--; @@ -708,5 +698,5 @@ if (dev->refcount == 0) { async_hangup(dev->sess); - hash_table_remove(&services, key, SERVICES_KEYS); + hash_table_remove(&services, &index); } @@ -732,10 +722,7 @@ if (type == LOC_OBJECT_SERVICE) { - unsigned long key[] = { - [SERVICES_KEY_HANDLE] = (unsigned long) index - }; - + fibril_mutex_lock(&services_mutex); - link_t *lnk = hash_table_find(&services, key); + ht_link_t *lnk = hash_table_find(&services, &index); if (lnk == NULL) { fibril_mutex_unlock(&services_mutex); @@ -743,5 +730,5 @@ } - service_t *dev = hash_table_get_instance(lnk, service_t, link); + service_t *dev = hash_table_get_inst(lnk, service_t, link); assert(dev->sess); Index: uspace/srv/fs/mfs/mfs.h =================================================================== --- uspace/srv/fs/mfs/mfs.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/mfs/mfs.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -142,5 +142,5 @@ unsigned refcnt; fs_node_t *fsnode; - link_t link; + ht_link_t link; }; Index: uspace/srv/fs/mfs/mfs_ops.c =================================================================== --- uspace/srv/fs/mfs/mfs_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/mfs/mfs_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -35,10 +35,7 @@ #include #include +#include #include "mfs.h" -#define OPEN_NODES_KEYS 2 -#define OPEN_NODES_SERVICE_KEY 0 -#define OPEN_NODES_INODE_KEY 1 -#define OPEN_NODES_BUCKETS 256 static bool check_magic_number(uint16_t magic, bool *native, @@ -61,8 +58,4 @@ static int mfs_unlink(fs_node_t *, fs_node_t *, const char *name); static int mfs_destroy_node(fs_node_t *fn); -static hash_index_t open_nodes_hash(unsigned long key[]); -static int open_nodes_compare(unsigned long key[], hash_count_t keys, - link_t *item); -static void open_nodes_remove_cb(link_t *link); static int mfs_node_get(fs_node_t **rfn, service_id_t service_id, fs_index_t index); @@ -95,38 +88,40 @@ /* Hash table interface for open nodes hash table */ -static hash_index_t -open_nodes_hash(unsigned long key[]) -{ - /* TODO: This is very simple and probably can be improved */ - return key[OPEN_NODES_INODE_KEY] % OPEN_NODES_BUCKETS; -} - -static int -open_nodes_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - struct mfs_node *mnode = hash_table_get_instance(item, struct mfs_node, link); - assert(keys > 0); - if (mnode->instance->service_id != - ((service_id_t) key[OPEN_NODES_SERVICE_KEY])) { - return false; - } - if (keys == 1) { - return true; - } - assert(keys == 2); - return (mnode->ino_i->index == key[OPEN_NODES_INODE_KEY]); -} - -static void -open_nodes_remove_cb(link_t *link) -{ - /* We don't use remove callback for this hash table */ -} - -static hash_table_operations_t open_nodes_ops = { + +typedef struct { + service_id_t service_id; + fs_index_t index; +} node_key_t; + +static size_t +open_nodes_key_hash(void *key) +{ + node_key_t *node_key = (node_key_t*)key; + return hash_combine(node_key->service_id, node_key->index); +} + +static size_t +open_nodes_hash(const ht_link_t *item) +{ + struct mfs_node *m = hash_table_get_inst(item, struct mfs_node, link); + return hash_combine(m->instance->service_id, m->ino_i->index); +} + +static bool +open_nodes_key_equal(void *key, const ht_link_t *item) +{ + node_key_t *node_key = (node_key_t*)key; + struct mfs_node *mnode = hash_table_get_inst(item, struct mfs_node, link); + + return node_key->service_id == mnode->instance->service_id + && node_key->index == mnode->ino_i->index; +} + +static hash_table_ops_t open_nodes_ops = { .hash = open_nodes_hash, - .compare = open_nodes_compare, - .remove_callback = open_nodes_remove_cb, + .key_hash = open_nodes_key_hash, + .key_equal = open_nodes_key_equal, + .equal = 0, + .remove_callback = 0, }; @@ -134,6 +129,5 @@ mfs_global_init(void) { - if (!hash_table_create(&open_nodes, OPEN_NODES_BUCKETS, - OPEN_NODES_KEYS, &open_nodes_ops)) { + if (!hash_table_create(&open_nodes, 0, 0, &open_nodes_ops)) { return ENOMEM; } @@ -406,13 +400,6 @@ mnode->refcnt = 1; - link_initialize(&mnode->link); - - unsigned long key[] = { - [OPEN_NODES_SERVICE_KEY] = inst->service_id, - [OPEN_NODES_INODE_KEY] = inum, - }; - fibril_mutex_lock(&open_nodes_lock); - hash_table_insert(&open_nodes, key, &mnode->link); + hash_table_insert(&open_nodes, &mnode->link); fibril_mutex_unlock(&open_nodes_lock); inst->open_nodes_cnt++; @@ -513,9 +500,5 @@ mnode->refcnt--; if (mnode->refcnt == 0) { - unsigned long key[] = { - [OPEN_NODES_SERVICE_KEY] = mnode->instance->service_id, - [OPEN_NODES_INODE_KEY] = mnode->ino_i->index - }; - hash_table_remove(&open_nodes, key, OPEN_NODES_KEYS); + hash_table_remove_item(&open_nodes, &mnode->link); assert(mnode->instance->open_nodes_cnt > 0); mnode->instance->open_nodes_cnt--; @@ -576,12 +559,13 @@ /* Check if the node is not already open */ - unsigned long key[] = { - [OPEN_NODES_SERVICE_KEY] = inst->service_id, - [OPEN_NODES_INODE_KEY] = index, + node_key_t key = { + .service_id = inst->service_id, + .index = index }; - link_t *already_open = hash_table_find(&open_nodes, key); + + ht_link_t *already_open = hash_table_find(&open_nodes, &key); if (already_open) { - mnode = hash_table_get_instance(already_open, struct mfs_node, link); + mnode = hash_table_get_inst(already_open, struct mfs_node, link); *rfn = mnode->fsnode; mnode->refcnt++; @@ -614,5 +598,4 @@ mnode->ino_i = ino_i; mnode->refcnt = 1; - link_initialize(&mnode->link); mnode->instance = inst; @@ -621,5 +604,5 @@ *rfn = node; - hash_table_insert(&open_nodes, key, &mnode->link); + hash_table_insert(&open_nodes, &mnode->link); inst->open_nodes_cnt++; Index: uspace/srv/fs/tmpfs/tmpfs.h =================================================================== --- uspace/srv/fs/tmpfs/tmpfs.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/tmpfs/tmpfs.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -62,5 +62,5 @@ fs_index_t index; /**< TMPFS node index. */ service_id_t service_id;/**< Service ID of block device. */ - link_t nh_link; /**< Nodes hash table link. */ + ht_link_t nh_link; /**< Nodes hash table link. */ tmpfs_dentry_type_t type; unsigned lnkcnt; /**< Link count. */ Index: uspace/srv/fs/tmpfs/tmpfs_ops.c =================================================================== --- uspace/srv/fs/tmpfs/tmpfs_ops.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/fs/tmpfs/tmpfs_ops.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -50,4 +50,5 @@ #include #include +#include #include #include @@ -55,6 +56,4 @@ #define min(a, b) ((a) < (b) ? (a) : (b)) #define max(a, b) ((a) > (b) ? (a) : (b)) - -#define NODES_BUCKETS 256 /** All root nodes have index 0. */ @@ -142,35 +141,36 @@ hash_table_t nodes; -#define NODES_KEY_DEV 0 -#define NODES_KEY_INDEX 1 - -/* Implementation of hash table interface for the nodes hash table. */ -static hash_index_t nodes_hash(unsigned long key[]) -{ - return key[NODES_KEY_INDEX] % NODES_BUCKETS; -} - -static int nodes_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - tmpfs_node_t *nodep = hash_table_get_instance(item, tmpfs_node_t, - nh_link); - - switch (keys) { - case 1: - return (nodep->service_id == key[NODES_KEY_DEV]); - case 2: - return ((nodep->service_id == key[NODES_KEY_DEV]) && - (nodep->index == key[NODES_KEY_INDEX])); - default: - assert((keys == 1) || (keys == 2)); - } - - return 0; -} - -static void nodes_remove_callback(link_t *item) -{ - tmpfs_node_t *nodep = hash_table_get_instance(item, tmpfs_node_t, - nh_link); +/* + * Implementation of hash table interface for the nodes hash table. + */ + +typedef struct { + service_id_t service_id; + fs_index_t index; +} node_key_t; + +static size_t nodes_key_hash(void *k) +{ + node_key_t *key = (node_key_t *)k; + return hash_combine(key->service_id, key->index); +} + +static size_t nodes_hash(const ht_link_t *item) +{ + tmpfs_node_t *nodep = hash_table_get_inst(item, tmpfs_node_t, nh_link); + return hash_combine(nodep->service_id, nodep->index); +} + +static bool nodes_key_equal(void *key_arg, const ht_link_t *item) +{ + tmpfs_node_t *node = hash_table_get_inst(item, tmpfs_node_t, nh_link); + node_key_t *key = (node_key_t *)key_arg; + + return key->service_id == node->service_id && key->index == node->index; +} + +static void nodes_remove_callback(ht_link_t *item) +{ + tmpfs_node_t *nodep = hash_table_get_inst(item, tmpfs_node_t, nh_link); while (!list_empty(&nodep->cs_list)) { @@ -192,7 +192,9 @@ /** TMPFS nodes hash table operations. */ -hash_table_operations_t nodes_ops = { +hash_table_ops_t nodes_ops = { .hash = nodes_hash, - .compare = nodes_compare, + .key_hash = nodes_key_hash, + .key_equal = nodes_key_equal, + .equal = 0, .remove_callback = nodes_remove_callback }; @@ -207,5 +209,4 @@ nodep->size = 0; nodep->data = NULL; - link_initialize(&nodep->nh_link); list_initialize(&nodep->cs_list); } @@ -220,5 +221,5 @@ bool tmpfs_init(void) { - if (!hash_table_create(&nodes, NODES_BUCKETS, 2, &nodes_ops)) + if (!hash_table_create(&nodes, 0, 0, &nodes_ops)) return false; @@ -238,17 +239,18 @@ } +static bool rm_service_id_nodes(ht_link_t *item, void *arg) +{ + service_id_t sid = *(service_id_t*)arg; + tmpfs_node_t *node = hash_table_get_inst(item, tmpfs_node_t, nh_link); + + if (node->service_id == sid) { + hash_table_remove_item(&nodes, &node->nh_link); + } + return true; +} + static void tmpfs_instance_done(service_id_t service_id) -{ - unsigned long key[] = { - [NODES_KEY_DEV] = service_id - }; - /* - * Here we are making use of one special feature of our hash table - * implementation, which allows to remove more items based on a partial - * key match. In the following, we are going to remove all nodes - * matching our device handle. The nodes_remove_callback() function will - * take care of resource deallocation. - */ - hash_table_remove(&nodes, key, 1); +{ + hash_table_apply(&nodes, rm_service_id_nodes, &service_id); } @@ -272,12 +274,14 @@ int tmpfs_node_get(fs_node_t **rfn, service_id_t service_id, fs_index_t index) { - unsigned long key[] = { - [NODES_KEY_DEV] = service_id, - [NODES_KEY_INDEX] = index + node_key_t key = { + .service_id = service_id, + .index = index }; - link_t *lnk = hash_table_find(&nodes, key); + + ht_link_t *lnk = hash_table_find(&nodes, &key); + if (lnk) { tmpfs_node_t *nodep; - nodep = hash_table_get_instance(lnk, tmpfs_node_t, nh_link); + nodep = hash_table_get_inst(lnk, tmpfs_node_t, nh_link); *rfn = FS_NODE(nodep); } else { @@ -331,9 +335,5 @@ /* Insert the new node into the nodes hash table. */ - unsigned long key[] = { - [NODES_KEY_DEV] = nodep->service_id, - [NODES_KEY_INDEX] = nodep->index - }; - hash_table_insert(&nodes, key, &nodep->nh_link); + hash_table_insert(&nodes, &nodep->nh_link); *rfn = FS_NODE(nodep); return EOK; @@ -346,10 +346,6 @@ assert(!nodep->lnkcnt); assert(list_empty(&nodep->cs_list)); - - unsigned long key[] = { - [NODES_KEY_DEV] = nodep->service_id, - [NODES_KEY_INDEX] = nodep->index - }; - hash_table_remove(&nodes, key, 2); + + hash_table_remove_item(&nodes, &nodep->nh_link); /* @@ -476,14 +472,14 @@ * Lookup the respective TMPFS node. */ - link_t *hlp; - unsigned long key[] = { - [NODES_KEY_DEV] = service_id, - [NODES_KEY_INDEX] = index + node_key_t key = { + .service_id = service_id, + .index = index }; - hlp = hash_table_find(&nodes, key); + + ht_link_t *hlp = hash_table_find(&nodes, &key); if (!hlp) return ENOENT; - tmpfs_node_t *nodep = hash_table_get_instance(hlp, tmpfs_node_t, - nh_link); + + tmpfs_node_t *nodep = hash_table_get_inst(hlp, tmpfs_node_t, nh_link); /* @@ -538,14 +534,15 @@ * Lookup the respective TMPFS node. */ - link_t *hlp; - unsigned long key[] = { - [NODES_KEY_DEV] = service_id, - [NODES_KEY_INDEX] = index + node_key_t key = { + .service_id = service_id, + .index = index }; - hlp = hash_table_find(&nodes, key); + + ht_link_t *hlp = hash_table_find(&nodes, &key); + if (!hlp) return ENOENT; - tmpfs_node_t *nodep = hash_table_get_instance(hlp, tmpfs_node_t, - nh_link); + + tmpfs_node_t *nodep = hash_table_get_inst(hlp, tmpfs_node_t, nh_link); /* @@ -600,12 +597,14 @@ * Lookup the respective TMPFS node. */ - unsigned long key[] = { - [NODES_KEY_DEV] = service_id, - [NODES_KEY_INDEX] = index + node_key_t key = { + .service_id = service_id, + .index = index }; - link_t *hlp = hash_table_find(&nodes, key); + + ht_link_t *hlp = hash_table_find(&nodes, &key); + if (!hlp) return ENOENT; - tmpfs_node_t *nodep = hash_table_get_instance(hlp, tmpfs_node_t, nh_link); + tmpfs_node_t *nodep = hash_table_get_inst(hlp, tmpfs_node_t, nh_link); if (size == nodep->size) @@ -636,13 +635,13 @@ static int tmpfs_destroy(service_id_t service_id, fs_index_t index) { - link_t *hlp; - unsigned long key[] = { - [NODES_KEY_DEV] = service_id, - [NODES_KEY_INDEX] = index + node_key_t key = { + .service_id = service_id, + .index = index }; - hlp = hash_table_find(&nodes, key); + + ht_link_t *hlp = hash_table_find(&nodes, &key); if (!hlp) return ENOENT; - tmpfs_node_t *nodep = hash_table_get_instance(hlp, tmpfs_node_t, + tmpfs_node_t *nodep = hash_table_get_inst(hlp, tmpfs_node_t, nh_link); return tmpfs_destroy_node(FS_NODE(nodep)); Index: uspace/srv/hid/input/generic/gsp.c =================================================================== --- uspace/srv/hid/input/generic/gsp.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/hid/input/generic/gsp.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -51,23 +51,44 @@ #include #include +#include #include #include -#define TRANS_TABLE_CHAINS 256 - /* - * Hash table operations for the transition function. - */ - -static hash_index_t trans_op_hash(unsigned long key[]); -static int trans_op_compare(unsigned long key[], hash_count_t keys, - link_t *item); -static void trans_op_remove_callback(link_t *item); - -static hash_table_operations_t trans_ops = { - .hash = trans_op_hash, - .compare = trans_op_compare, - .remove_callback = trans_op_remove_callback + * Transition function hash table operations. + */ +typedef struct { + int old_state; + int input; +} trans_key_t; + +static size_t trans_key_hash(void *key) +{ + trans_key_t *trans_key = (trans_key_t *)key; + return hash_combine(trans_key->input, trans_key->old_state); +} + +static size_t trans_hash(const ht_link_t *item) +{ + gsp_trans_t *t = hash_table_get_inst(item, gsp_trans_t, link); + return hash_combine(t->input, t->old_state); +} + +static bool trans_key_equal(void *key, const ht_link_t *item) +{ + trans_key_t *trans_key = (trans_key_t *)key; + gsp_trans_t *t = hash_table_get_inst(item, gsp_trans_t, link); + + return trans_key->input == t->input && trans_key->old_state == t->old_state; +} + +static hash_table_ops_t trans_ops = { + .hash = trans_hash, + .key_hash = trans_key_hash, + .key_equal = trans_key_equal, + .equal = 0, + .remove_callback = 0 }; + static gsp_trans_t *trans_lookup(gsp_t *p, int state, int input); @@ -75,9 +96,9 @@ static gsp_trans_t *trans_new(void); -/** Initialise scancode parser. */ +/** Initialize scancode parser. */ void gsp_init(gsp_t *p) { p->states = 1; - hash_table_create(&p->trans, TRANS_TABLE_CHAINS, 2, &trans_ops); + hash_table_create(&p->trans, 0, 0, &trans_ops); } @@ -223,14 +244,15 @@ static gsp_trans_t *trans_lookup(gsp_t *p, int state, int input) { - link_t *item; - unsigned long key[2]; - - key[0] = state; - key[1] = input; - - item = hash_table_find(&p->trans, key); + ht_link_t *item; + + trans_key_t key = { + .input = input, + .old_state = state + }; + + item = hash_table_find(&p->trans, &key); if (item == NULL) return NULL; - return hash_table_get_instance(item, gsp_trans_t, link); + return hash_table_get_inst(item, gsp_trans_t, link); } @@ -242,10 +264,5 @@ static void trans_insert(gsp_t *p, gsp_trans_t *t) { - unsigned long key[2]; - - key[0] = t->old_state; - key[1] = t->input; - - hash_table_insert(&p->trans, key, &t->link); + hash_table_insert(&p->trans, &t->link); } @@ -264,26 +281,4 @@ } -/* - * Transition function hash table operations. - */ - -static hash_index_t trans_op_hash(unsigned long key[]) -{ - return (key[0] * 17 + key[1]) % TRANS_TABLE_CHAINS; -} - -static int trans_op_compare(unsigned long key[], hash_count_t keys, - link_t *item) -{ - gsp_trans_t *t; - - t = hash_table_get_instance(item, gsp_trans_t, link); - return ((key[0] == (unsigned long) t->old_state) - && (key[1] == (unsigned long) t->input)); -} - -static void trans_op_remove_callback(link_t *item) -{ -} /** Index: uspace/srv/hid/input/include/gsp.h =================================================================== --- uspace/srv/hid/input/include/gsp.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/hid/input/include/gsp.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -56,5 +56,5 @@ /** Scancode parser transition. */ typedef struct { - link_t link; /**< Link to hash table in @c gsp_t */ + ht_link_t link; /**< Link to hash table in @c gsp_t */ /* Preconditions */ Index: uspace/srv/ns/service.c =================================================================== --- uspace/srv/ns/service.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/ns/service.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -40,9 +40,8 @@ #include "ns.h" -#define SERVICE_HASH_TABLE_CHAINS 20 /** Service hash table item. */ typedef struct { - link_t link; + ht_link_t link; sysarg_t service; /**< Service ID. */ sysarg_t phone; /**< Phone registered with the service. */ @@ -50,63 +49,29 @@ } hashed_service_t; -/** Compute hash index into service hash table. - * - * @param key Pointer keys. However, only the first key (i.e. service number) - * is used to compute the hash index. - * - * @return Hash index corresponding to key[0]. - * - */ -static hash_index_t service_hash(unsigned long key[]) + +static size_t service_key_hash(void *key) { - assert(key); - return (key[0] % SERVICE_HASH_TABLE_CHAINS); + return *(sysarg_t*)key; } -/** Compare a key with hashed item. - * - * This compare function always ignores the third key. - * It exists only to make it possible to remove records - * originating from connection with key[1] in_phone_hash - * value. Note that this is close to being classified - * as a nasty hack. - * - * @param key Array of keys. - * @param keys Must be lesser or equal to 3. - * @param item Pointer to a hash table item. - * - * @return Non-zero if the key matches the item, zero otherwise. - * - */ -static int service_compare(unsigned long key[], hash_count_t keys, link_t *item) +static size_t service_hash(const ht_link_t *item) { - assert(key); - assert(keys <= 3); - assert(item); - - hashed_service_t *hs = hash_table_get_instance(item, hashed_service_t, link); - - if (keys == 2) - return ((key[0] == hs->service) && (key[1] == hs->in_phone_hash)); - else - return (key[0] == hs->service); + hashed_service_t *hs = hash_table_get_inst(item, hashed_service_t, link); + return hs->service; } -/** Perform actions after removal of item from the hash table. - * - * @param item Item that was removed from the hash table. - * - */ -static void service_remove(link_t *item) +static bool service_key_equal(void *key, const ht_link_t *item) { - assert(item); - free(hash_table_get_instance(item, hashed_service_t, link)); + hashed_service_t *hs = hash_table_get_inst(item, hashed_service_t, link); + return hs->service == *(sysarg_t*)key; } /** Operations for service hash table. */ -static hash_table_operations_t service_hash_table_ops = { +static hash_table_ops_t service_hash_table_ops = { .hash = service_hash, - .compare = service_compare, - .remove_callback = service_remove + .key_hash = service_key_hash, + .key_equal = service_key_equal, + .equal = 0, + .remove_callback = 0 }; @@ -127,6 +92,5 @@ int service_init(void) { - if (!hash_table_create(&service_hash_table, SERVICE_HASH_TABLE_CHAINS, - 3, &service_hash_table_ops)) { + if (!hash_table_create(&service_hash_table, 0, 0, &service_hash_table_ops)) { printf(NAME ": No memory available for services\n"); return ENOMEM; @@ -145,15 +109,9 @@ pending_conn_t *pr = list_get_instance(cur, pending_conn_t, link); - unsigned long keys[3] = { - pr->service, - 0, - 0 - }; - - link_t *link = hash_table_find(&service_hash_table, keys); + ht_link_t *link = hash_table_find(&service_hash_table, &pr->service); if (!link) continue; - hashed_service_t *hs = hash_table_get_instance(link, hashed_service_t, link); + hashed_service_t *hs = hash_table_get_inst(link, hashed_service_t, link); (void) ipc_forward_fast(pr->callid, hs->phone, pr->arg2, pr->arg3, 0, IPC_FF_NONE); @@ -176,11 +134,5 @@ int register_service(sysarg_t service, sysarg_t phone, ipc_call_t *call) { - unsigned long keys[3] = { - service, - call->in_phone_hash, - 0 - }; - - if (hash_table_find(&service_hash_table, keys)) + if (hash_table_find(&service_hash_table, &service)) return EEXISTS; @@ -189,9 +141,8 @@ return ENOMEM; - link_initialize(&hs->link); hs->service = service; hs->phone = phone; hs->in_phone_hash = call->in_phone_hash; - hash_table_insert(&service_hash_table, keys, &hs->link); + hash_table_insert(&service_hash_table, &hs->link); return EOK; @@ -210,11 +161,6 @@ { sysarg_t retval; - unsigned long keys[3] = { - service, - 0, - 0 - }; - link_t *link = hash_table_find(&service_hash_table, keys); + ht_link_t *link = hash_table_find(&service_hash_table, &service); if (!link) { if (IPC_GET_ARG4(*call) & IPC_FLAG_BLOCKING) { @@ -239,5 +185,5 @@ } - hashed_service_t *hs = hash_table_get_instance(link, hashed_service_t, link); + hashed_service_t *hs = hash_table_get_inst(link, hashed_service_t, link); (void) ipc_forward_fast(callid, hs->phone, IPC_GET_ARG2(*call), IPC_GET_ARG3(*call), 0, IPC_FF_NONE); Index: uspace/srv/ns/task.c =================================================================== --- uspace/srv/ns/task.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/ns/task.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -43,6 +43,4 @@ #include "ns.h" -#define TASK_HASH_TABLE_CHAINS 256 -#define P2I_HASH_TABLE_CHAINS 256 /* TODO: @@ -55,5 +53,5 @@ /** Task hash table item. */ typedef struct { - link_t link; + ht_link_t link; task_id_t id; /**< Task ID. */ @@ -63,57 +61,34 @@ } hashed_task_t; -/** Compute hash index into task hash table. - * - * @param key Pointer keys. However, only the first key (i.e. truncated task - * number) is used to compute the hash index. - * - * @return Hash index corresponding to key[0]. - * - */ -static hash_index_t task_hash(unsigned long key[]) -{ - assert(key); - return (LOWER32(key[0]) % TASK_HASH_TABLE_CHAINS); -} - -/** Compare a key with hashed item. - * - * @param key Array of keys. - * @param keys Must be less than or equal to 2. - * @param item Pointer to a hash table item. - * - * @return Non-zero if the key matches the item, zero otherwise. - * - */ -static int task_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - assert(key); - assert(keys <= 2); - assert(item); - - hashed_task_t *ht = hash_table_get_instance(item, hashed_task_t, link); - - if (keys == 2) - return ((LOWER32(key[1]) == UPPER32(ht->id)) - && (LOWER32(key[0]) == LOWER32(ht->id))); - else - return (LOWER32(key[0]) == LOWER32(ht->id)); -} - -/** Perform actions after removal of item from the hash table. - * - * @param item Item that was removed from the hash table. - * - */ -static void task_remove(link_t *item) -{ - assert(item); - free(hash_table_get_instance(item, hashed_task_t, link)); + +static size_t task_key_hash(void *key) +{ + return *(task_id_t*)key; +} + +static size_t task_hash(const ht_link_t *item) +{ + hashed_task_t *ht = hash_table_get_inst(item, hashed_task_t, link); + return ht->id; +} + +static bool task_key_equal(void *key, const ht_link_t *item) +{ + hashed_task_t *ht = hash_table_get_inst(item, hashed_task_t, link); + return ht->id == *(task_id_t*)key; +} + +/** Perform actions after removal of item from the hash table. */ +static void task_remove(ht_link_t *item) +{ + free(hash_table_get_inst(item, hashed_task_t, link)); } /** Operations for task hash table. */ -static hash_table_operations_t task_hash_table_ops = { +static hash_table_ops_t task_hash_table_ops = { .hash = task_hash, - .compare = task_compare, + .key_hash = task_key_hash, + .key_equal = task_key_equal, + .equal = 0, .remove_callback = task_remove }; @@ -123,57 +98,48 @@ typedef struct { - link_t link; + ht_link_t link; sysarg_t in_phone_hash; /**< Incoming phone hash. */ task_id_t id; /**< Task ID. */ } p2i_entry_t; -/** Compute hash index into task hash table. - * - * @param key Array of keys. - * - * @return Hash index corresponding to key[0]. - * - */ -static hash_index_t p2i_hash(unsigned long key[]) -{ - assert(key); - return (key[0] % TASK_HASH_TABLE_CHAINS); -} - -/** Compare a key with hashed item. - * - * @param key Array of keys. - * @param keys Must be less than or equal to 1. - * @param item Pointer to a hash table item. - * - * @return Non-zero if the key matches the item, zero otherwise. - * - */ -static int p2i_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - assert(key); - assert(keys == 1); +/* phone-to-id hash table operations */ + +static size_t p2i_key_hash(void *key) +{ + sysarg_t in_phone_hash = *(sysarg_t*)key; + return in_phone_hash; +} + +static size_t p2i_hash(const ht_link_t *item) +{ + p2i_entry_t *entry = hash_table_get_inst(item, p2i_entry_t, link); + return entry->in_phone_hash; +} + +static bool p2i_key_equal(void *key, const ht_link_t *item) +{ + sysarg_t in_phone_hash = *(sysarg_t*)key; + p2i_entry_t *entry = hash_table_get_inst(item, p2i_entry_t, link); + + return (in_phone_hash == entry->in_phone_hash); +} + +/** Perform actions after removal of item from the hash table. + * + * @param item Item that was removed from the hash table. + * + */ +static void p2i_remove(ht_link_t *item) +{ assert(item); - - p2i_entry_t *entry = hash_table_get_instance(item, p2i_entry_t, link); - - return (key[0] == entry->in_phone_hash); -} - -/** Perform actions after removal of item from the hash table. - * - * @param item Item that was removed from the hash table. - * - */ -static void p2i_remove(link_t *item) -{ - assert(item); - free(hash_table_get_instance(item, p2i_entry_t, link)); + free(hash_table_get_inst(item, p2i_entry_t, link)); } /** Operations for task hash table. */ -static hash_table_operations_t p2i_ops = { +static hash_table_ops_t p2i_ops = { .hash = p2i_hash, - .compare = p2i_compare, + .key_hash = p2i_key_hash, + .key_equal = p2i_key_equal, + .equal = 0, .remove_callback = p2i_remove }; @@ -193,12 +159,10 @@ int task_init(void) { - if (!hash_table_create(&task_hash_table, TASK_HASH_TABLE_CHAINS, - 2, &task_hash_table_ops)) { + if (!hash_table_create(&task_hash_table, 0, 0, &task_hash_table_ops)) { printf(NAME ": No memory available for tasks\n"); return ENOMEM; } - if (!hash_table_create(&phone_to_id, P2I_HASH_TABLE_CHAINS, - 1, &p2i_ops)) { + if (!hash_table_create(&phone_to_id, 0, 0, &p2i_ops)) { printf(NAME ": No memory available for tasks\n"); return ENOMEM; @@ -218,14 +182,9 @@ pending_wait_t *pr = list_get_instance(cur, pending_wait_t, link); - unsigned long keys[2] = { - LOWER32(pr->id), - UPPER32(pr->id) - }; - - link_t *link = hash_table_find(&task_hash_table, keys); + ht_link_t *link = hash_table_find(&task_hash_table, &pr->id); if (!link) continue; - hashed_task_t *ht = hash_table_get_instance(link, hashed_task_t, link); + hashed_task_t *ht = hash_table_get_inst(link, hashed_task_t, link); if (!ht->finished) continue; @@ -238,5 +197,5 @@ } - hash_table_remove(&task_hash_table, keys, 2); + hash_table_remove(&task_hash_table, &pr->id); list_remove(cur); free(pr); @@ -250,12 +209,7 @@ task_exit_t texit; - unsigned long keys[2] = { - LOWER32(id), - UPPER32(id) - }; - - link_t *link = hash_table_find(&task_hash_table, keys); + ht_link_t *link = hash_table_find(&task_hash_table, &id); hashed_task_t *ht = (link != NULL) ? - hash_table_get_instance(link, hashed_task_t, link) : NULL; + hash_table_get_inst(link, hashed_task_t, link) : NULL; if (ht == NULL) { @@ -281,5 +235,5 @@ } - hash_table_remove(&task_hash_table, keys, 2); + hash_table_remove_item(&task_hash_table, link); retval = EOK; @@ -293,10 +247,8 @@ int ns_task_id_intro(ipc_call_t *call) { - unsigned long keys[2]; task_id_t id = MERGE_LOUP32(IPC_GET_ARG1(*call), IPC_GET_ARG2(*call)); - keys[0] = call->in_phone_hash; - - link_t *link = hash_table_find(&phone_to_id, keys); + + ht_link_t *link = hash_table_find(&phone_to_id, &call->in_phone_hash); if (link != NULL) return EEXISTS; @@ -314,8 +266,7 @@ */ - link_initialize(&entry->link); entry->in_phone_hash = call->in_phone_hash; entry->id = id; - hash_table_insert(&phone_to_id, keys, &entry->link); + hash_table_insert(&phone_to_id, &entry->link); /* @@ -323,13 +274,9 @@ */ - keys[0] = LOWER32(id); - keys[1] = UPPER32(id); - - link_initialize(&ht->link); ht->id = id; ht->finished = false; ht->have_rval = false; ht->retval = -1; - hash_table_insert(&task_hash_table, keys, &ht->link); + hash_table_insert(&task_hash_table, &ht->link); return EOK; @@ -338,11 +285,9 @@ static int get_id_by_phone(sysarg_t phone_hash, task_id_t *id) { - unsigned long keys[1] = {phone_hash}; - - link_t *link = hash_table_find(&phone_to_id, keys); + ht_link_t *link = hash_table_find(&phone_to_id, &phone_hash); if (link == NULL) return ENOENT; - p2i_entry_t *entry = hash_table_get_instance(link, p2i_entry_t, link); + p2i_entry_t *entry = hash_table_get_inst(link, p2i_entry_t, link); *id = entry->id; @@ -357,12 +302,7 @@ return rc; - unsigned long keys[2] = { - LOWER32(id), - UPPER32(id) - }; - - link_t *link = hash_table_find(&task_hash_table, keys); + ht_link_t *link = hash_table_find(&task_hash_table, &id); hashed_task_t *ht = (link != NULL) ? - hash_table_get_instance(link, hashed_task_t, link) : NULL; + hash_table_get_inst(link, hashed_task_t, link) : NULL; if ((ht == NULL) || (ht->finished)) @@ -378,6 +318,4 @@ int ns_task_disconnect(ipc_call_t *call) { - unsigned long keys[2]; - task_id_t id; int rc = get_id_by_phone(call->in_phone_hash, &id); @@ -386,16 +324,12 @@ /* Delete from phone-to-id map. */ - keys[0] = call->in_phone_hash; - hash_table_remove(&phone_to_id, keys, 1); + hash_table_remove(&phone_to_id, &call->in_phone_hash); /* Mark task as finished. */ - keys[0] = LOWER32(id); - keys[1] = UPPER32(id); - - link_t *link = hash_table_find(&task_hash_table, keys); - hashed_task_t *ht = - hash_table_get_instance(link, hashed_task_t, link); - if (ht == NULL) + ht_link_t *link = hash_table_find(&task_hash_table, &id); + if (link == NULL) return EOK; + + hashed_task_t *ht = hash_table_get_inst(link, hashed_task_t, link); ht->finished = true; Index: uspace/srv/vfs/vfs.h =================================================================== --- uspace/srv/vfs/vfs.h (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/vfs/vfs.h (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -36,4 +36,5 @@ #include #include +#include #include #include @@ -112,5 +113,5 @@ unsigned lnkcnt; - link_t nh_link; /**< Node hash-table link. */ + ht_link_t nh_link; /**< Node hash-table link. */ vfs_node_type_t type; /**< Partial info about the node type. */ Index: uspace/srv/vfs/vfs_node.c =================================================================== --- uspace/srv/vfs/vfs_node.c (revision 1d01cca194390f18b0c95a74d05d16aeffa88ab8) +++ uspace/srv/vfs/vfs_node.c (revision da68871a1ca8b07736e941032a954a3f19eadccc) @@ -41,4 +41,5 @@ #include #include +#include #include #include @@ -58,13 +59,16 @@ #define KEY_INDEX 2 -static hash_index_t nodes_hash(unsigned long []); -static int nodes_compare(unsigned long [], hash_count_t, link_t *); -static void nodes_remove_callback(link_t *); +static size_t nodes_key_hash(void *); +static size_t nodes_hash(const ht_link_t *); +static bool nodes_key_equal(void *, const ht_link_t *); +static vfs_triplet_t node_triplet(vfs_node_t *node); /** VFS node hash table operations. */ -hash_table_operations_t nodes_ops = { +hash_table_ops_t nodes_ops = { .hash = nodes_hash, - .compare = nodes_compare, - .remove_callback = nodes_remove_callback + .key_hash = nodes_key_hash, + .key_equal = nodes_key_equal, + .equal = 0, + .remove_callback = 0, }; @@ -75,5 +79,5 @@ bool vfs_nodes_init(void) { - return hash_table_create(&nodes, NODES_BUCKETS, 3, &nodes_ops); + return hash_table_create(&nodes, 0, 0, &nodes_ops); } @@ -114,11 +118,5 @@ */ - unsigned long key[] = { - [KEY_FS_HANDLE] = node->fs_handle, - [KEY_DEV_HANDLE] = node->service_id, - [KEY_INDEX] = node->index - }; - - hash_table_remove(&nodes, key, 3); + hash_table_remove_item(&nodes, &node->nh_link); free_vfs_node = true; @@ -158,10 +156,5 @@ { fibril_mutex_lock(&nodes_mutex); - unsigned long key[] = { - [KEY_FS_HANDLE] = node->fs_handle, - [KEY_DEV_HANDLE] = node->service_id, - [KEY_INDEX] = node->index - }; - hash_table_remove(&nodes, key, 3); + hash_table_remove_item(&nodes, &node->nh_link); fibril_mutex_unlock(&nodes_mutex); free(node); @@ -182,14 +175,8 @@ vfs_node_t *vfs_node_get(vfs_lookup_res_t *result) { - unsigned long key[] = { - [KEY_FS_HANDLE] = result->triplet.fs_handle, - [KEY_DEV_HANDLE] = result->triplet.service_id, - [KEY_INDEX] = result->triplet.index - }; - link_t *tmp; vfs_node_t *node; fibril_mutex_lock(&nodes_mutex); - tmp = hash_table_find(&nodes, key); + ht_link_t *tmp = hash_table_find(&nodes, &result->triplet); if (!tmp) { node = (vfs_node_t *) malloc(sizeof(vfs_node_t)); @@ -205,9 +192,8 @@ node->lnkcnt = result->lnkcnt; node->type = result->type; - link_initialize(&node->nh_link); fibril_rwlock_initialize(&node->contents_rwlock); - hash_table_insert(&nodes, key, &node->nh_link); + hash_table_insert(&nodes, &node->nh_link); } else { - node = hash_table_get_instance(tmp, vfs_node_t, nh_link); + node = hash_table_get_inst(tmp, vfs_node_t, nh_link); if (node->type == VFS_NODE_UNKNOWN && result->type != VFS_NODE_UNKNOWN) { @@ -240,24 +226,4 @@ } -hash_index_t nodes_hash(unsigned long key[]) -{ - hash_index_t a = key[KEY_FS_HANDLE] << (NODES_BUCKETS_LOG / 4); - hash_index_t b = (a | key[KEY_DEV_HANDLE]) << (NODES_BUCKETS_LOG / 2); - - return (b | key[KEY_INDEX]) & (NODES_BUCKETS - 1); -} - -int nodes_compare(unsigned long key[], hash_count_t keys, link_t *item) -{ - vfs_node_t *node = hash_table_get_instance(item, vfs_node_t, nh_link); - return (node->fs_handle == (fs_handle_t) key[KEY_FS_HANDLE]) && - (node->service_id == key[KEY_DEV_HANDLE]) && - (node->index == key[KEY_INDEX]); -} - -void nodes_remove_callback(link_t *item) -{ -} - struct refcnt_data { /** Sum of all reference counts for this file system instance. */ @@ -267,7 +233,7 @@ }; -static void refcnt_visitor(link_t *item, void *arg) -{ - vfs_node_t *node = hash_table_get_instance(item, vfs_node_t, nh_link); +static bool refcnt_visitor(ht_link_t *item, void *arg) +{ + vfs_node_t *node = hash_table_get_inst(item, vfs_node_t, nh_link); struct refcnt_data *rd = (void *) arg; @@ -275,4 +241,6 @@ (node->service_id == rd->service_id)) rd->refcnt += node->refcnt; + + return true; } @@ -315,4 +283,39 @@ } + +static size_t nodes_key_hash(void *key) +{ + vfs_triplet_t *tri = key; + size_t hash = hash_combine(tri->fs_handle, tri->index); + return hash_combine(hash, tri->service_id); +} + +static size_t nodes_hash(const ht_link_t *item) +{ + vfs_node_t *node = hash_table_get_inst(item, vfs_node_t, nh_link); + vfs_triplet_t tri = node_triplet(node); + return nodes_key_hash(&tri); +} + +static bool nodes_key_equal(void *key, const ht_link_t *item) +{ + vfs_triplet_t *tri = key; + vfs_node_t *node = hash_table_get_inst(item, vfs_node_t, nh_link); + return node->fs_handle == tri->fs_handle + && node->service_id == tri->service_id + && node->index == tri->index; +} + +static inline vfs_triplet_t node_triplet(vfs_node_t *node) +{ + vfs_triplet_t tri = { + .fs_handle = node->fs_handle, + .service_id = node->service_id, + .index = node->index + }; + + return tri; +} + /** * @}