Changeset 8e3ed06 in mainline for kernel/generic/include/synch/rcu.h


Ignore:
Timestamp:
2012-07-29T17:28:45Z (12 years ago)
Author:
Adam Hraska <adam.hraska+hos@…>
Branches:
lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
Children:
5b03a72
Parents:
d99fac9
Message:

rcu: Allowed inlining of the RCU read side.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • kernel/generic/include/synch/rcu.h

    rd99fac9 r8e3ed06  
    3636#define KERN_RCU_H_
    3737
    38 #include <adt/list.h>
    39 #include <synch/semaphore.h>
     38#include <synch/rcu_types.h>
    4039#include <compiler/barrier.h>
    4140
    4241
    43 /* Fwd decl. */
    44 struct thread;
    45 struct rcu_item;
    46 
    47 /** Grace period number typedef. */
    48 typedef uint64_t rcu_gp_t;
    49 
    50 /** RCU callback type. The passed rcu_item_t maybe freed. */
    51 typedef void (*rcu_func_t)(struct rcu_item *rcu_item);
    52 
    53 typedef struct rcu_item {
    54         rcu_func_t func;
    55         struct rcu_item *next;
    56 } rcu_item_t;
    57 
    58 
    59 /** RCU related per-cpu data. */
    60 typedef struct rcu_cpu_data {
    61         /** The cpu recorded a quiescent state last time during this grace period */
    62         rcu_gp_t last_seen_gp;
    63        
    64         /** Pointer to the currently used nesting count (THREAD's or CPU's). */
    65         size_t *pnesting_cnt;
    66         /** Temporary nesting count if THREAD is NULL, eg in scheduler(). */
    67         size_t tmp_nesting_cnt;
    68 
    69         /** Callbacks to invoke once the current grace period ends, ie cur_cbs_gp.
    70          * Accessed by the local reclaimer only.
    71          */
    72         rcu_item_t *cur_cbs;
    73         /** Number of callbacks in cur_cbs. */
    74         size_t cur_cbs_cnt;
    75         /** Callbacks to invoke once the next grace period ends, ie next_cbs_gp.
    76          * Accessed by the local reclaimer only.
    77          */
    78         rcu_item_t *next_cbs;
    79         /** Number of callbacks in next_cbs. */
    80         size_t next_cbs_cnt;
    81         /** New callbacks are place at the end of this list. */
    82         rcu_item_t *arriving_cbs;
    83         /** Tail of arriving_cbs list. Disable interrupts to access. */
    84         rcu_item_t **parriving_cbs_tail;
    85         /** Number of callbacks currently in arriving_cbs.
    86          * Disable interrupts to access.
    87          */
    88         size_t arriving_cbs_cnt;
    89 
    90         /** At the end of this grace period callbacks in cur_cbs will be invoked.*/
    91         rcu_gp_t cur_cbs_gp;
    92         /** At the end of this grace period callbacks in next_cbs will be invoked.
    93          *
    94          * Should be the next grace period but it allows the reclaimer to
    95          * notice if it missed a grace period end announcement. In that
    96          * case it can execute next_cbs without waiting for another GP.
    97          *
    98          * Invariant: next_cbs_gp >= cur_cbs_gp
    99          */
    100         rcu_gp_t next_cbs_gp;
    101        
    102         /** This cpu has not yet passed a quiescent state and it is delaying the
    103          * detector. Once it reaches a QS it must sema_up(rcu.remaining_readers).
    104          */
    105         bool is_delaying_gp;
    106        
    107         /** Positive if there are callbacks pending in arriving_cbs. */
    108         semaphore_t arrived_flag;
    109        
    110         /** The reclaimer should expedite GPs for cbs in arriving_cbs. */
    111         bool expedite_arriving;
    112        
    113         /** Protected by global rcu.barrier_mtx. */
    114         rcu_item_t barrier_item;
    115        
    116         /** Interruptable attached reclaimer thread. */
    117         struct thread *reclaimer_thr;
    118        
    119         /* Some statistics. */
    120         size_t stat_max_cbs;
    121         size_t stat_avg_cbs;
    122         size_t stat_missed_gps;
    123         size_t stat_missed_gp_in_wait;
    124         size_t stat_max_slice_cbs;
    125         size_t last_arriving_cnt;
    126 } rcu_cpu_data_t;
    127 
    128 
    129 /** RCU related per-thread data. */
    130 typedef struct rcu_thread_data {
    131         /** The number of times an RCU reader section is nested.
    132          *
    133          * If positive, it is definitely executing reader code. If zero,
    134          * the thread might already be executing reader code thanks to
    135          * cpu instruction reordering.
    136          */
    137         size_t nesting_cnt;
    138        
    139         /** True if the thread was preempted in a reader section.
    140          *
    141          * The thread is place into rcu.cur_preempted or rcu.next_preempted
    142          * and must remove itself in rcu_read_unlock().
    143          *
    144          * Access with interrupts disabled.
    145          */
    146         bool was_preempted;
    147         /** Preempted threads link. Access with rcu.prempt_lock.*/
    148         link_t preempt_link;
    149 } rcu_thread_data_t;
    15042
    15143
    … …  
    20193#define rcu_access(ptr) ACCESS_ONCE(ptr)
    20294
    203 extern void rcu_read_lock(void);
    204 extern void rcu_read_unlock(void);
     95
     96
     97
     98#include <debug.h>
     99#include <preemption.h>
     100#include <cpu.h>
     101#include <proc/thread.h>
     102
     103
    205104extern bool rcu_read_locked(void);
    206105extern void rcu_synchronize(void);
    … …  
    224123extern void _rcu_synchronize(bool expedite);
    225124
     125
     126/* Fwd decl. required by the inlined implementation. Not part of public API. */
     127extern rcu_gp_t _rcu_cur_gp;
     128extern void _rcu_signal_read_unlock(void);
     129
     130
     131/** Unconditionally records a quiescent state for the local cpu. */
     132static inline void _rcu_record_qs(void)
     133{
     134        ASSERT(PREEMPTION_DISABLED || interrupts_disabled());
     135       
     136        /*
     137         * A new GP was started since the last time we passed a QS.
     138         * Notify the detector we have reached a new QS.
     139         */
     140        if (CPU->rcu.last_seen_gp != _rcu_cur_gp) {
     141                rcu_gp_t cur_gp = ACCESS_ONCE(_rcu_cur_gp);
     142                /*
     143                 * Contain memory accesses within a reader critical section.
     144                 * If we are in rcu_lock() it also makes changes prior to the
     145                 * start of the GP visible in the reader section.
     146                 */
     147                memory_barrier();
     148                /*
     149                 * Acknowledge we passed a QS since the beginning of rcu.cur_gp.
     150                 * Cache coherency will lazily transport the value to the
     151                 * detector while it sleeps in gp_sleep().
     152                 *
     153                 * Note that there is a theoretical possibility that we
     154                 * overwrite a more recent/greater last_seen_gp here with
     155                 * an older/smaller value. If this cpu is interrupted here
     156                 * while in rcu_lock() reader sections in the interrupt handler
     157                 * will update last_seen_gp to the same value as is currently
     158                 * in local cur_gp. However, if the cpu continues processing
     159                 * interrupts and the detector starts a new GP immediately,
     160                 * local interrupt handlers may update last_seen_gp again (ie
     161                 * properly ack the new GP) with a value greater than local cur_gp.
     162                 * Resetting last_seen_gp to a previous value here is however
     163                 * benign and we only have to remember that this reader may end up
     164                 * in cur_preempted even after the GP ends. That is why we
     165                 * append next_preempted to cur_preempted rather than overwriting
     166                 * it as if cur_preempted were empty.
     167                 */
     168                CPU->rcu.last_seen_gp = cur_gp;
     169        }
     170}
     171
     172/** Delimits the start of an RCU reader critical section.
     173 *
     174 * Reader sections may be nested and are preemptable. You must not
     175 * however block/sleep within reader sections.
     176 */
     177static inline void rcu_read_lock(void)
     178{
     179        ASSERT(CPU);
     180        preemption_disable();
     181
     182        /* Record a QS if not in a reader critical section. */
     183        if (0 == *CPU->rcu.pnesting_cnt)
     184                _rcu_record_qs();
     185
     186        ++(*CPU->rcu.pnesting_cnt);
     187
     188        preemption_enable();
     189}
     190
     191/** Delimits the end of an RCU reader critical section. */
     192static inline void rcu_read_unlock(void)
     193{
     194        ASSERT(CPU);
     195        preemption_disable();
     196       
     197        if (0 == --(*CPU->rcu.pnesting_cnt)) {
     198                _rcu_record_qs();
     199               
     200                /*
     201                 * The thread was preempted while in a critical section or
     202                 * the detector is eagerly waiting for this cpu's reader
     203                 * to finish.
     204                 *
     205                 * Note that THREAD may be 0 in scheduler() and not just during boot.
     206                 */
     207                if ((THREAD && THREAD->rcu.was_preempted) || CPU->rcu.is_delaying_gp) {
     208                        /* Rechecks with disabled interrupts. */
     209                        _rcu_signal_read_unlock();
     210                }
     211        }
     212
     213       
     214        preemption_enable();
     215}
     216
     217
    226218#endif
    227219
Note: See TracChangeset for help on using the changeset viewer.