Changeset 0cf813d in mainline


Ignore:
Timestamp:
2012-07-16T15:37:11Z (12 years ago)
Author:
Adam Hraska <adam.hraska+hos@…>
Branches:
lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
Children:
1c1da4b
Parents:
057e77f
Message:

rcu: Added new statistics. Changed reclaimers to run callbacks with preemption disabled if too many callbacks are pending in order to avoid exhausting system memory.

Location:
kernel/generic
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • kernel/generic/include/compiler/barrier.h

    r057e77f r0cf813d  
    55#define compiler_barrier() asm volatile ("" ::: "memory")
    66
     7/** Forces the compiler to access (ie load/store) the variable only once. */
    78#define ACCESS_ONCE(var) (*((volatile typeof(var)*)&(var)))
    89

  • kernel/generic/include/synch/rcu.h

    r057e77f r0cf813d  
    7171         */
    7272        rcu_item_t *cur_cbs;
     73        /** Number of callbacks in cur_cbs. */
     74        size_t cur_cbs_cnt;
    7375        /** Callbacks to invoke once the next grace period ends, ie next_cbs_gp.
    7476         * Accessed by the local reclaimer only.
    7577         */
    7678        rcu_item_t *next_cbs;
     79        /** Number of callbacks in next_cbs. */
     80        size_t next_cbs_cnt;
    7781        /** New callbacks are place at the end of this list. */
    7882        rcu_item_t *arriving_cbs;
    … …  
    114118        size_t stat_avg_cbs;
    115119        size_t stat_missed_gps;
     120        size_t stat_missed_gp_in_wait;
     121        size_t stat_max_slice_cbs;
     122        size_t last_arriving_cnt;
    116123} rcu_cpu_data_t;
    117124

  • kernel/generic/src/synch/rcu.c

    r057e77f r0cf813d  
    5454 * when non-expedited grace period detection is in progress.
    5555 */
    56 #define DETECT_SLEEP_MS    5
     56#define DETECT_SLEEP_MS    10
    5757/*
    5858 * Max number of pending callbacks in the local cpu's queue before
    5959 * aggressively expediting the current grace period
    6060 */
    61 #define EXPEDITE_THRESHOLD 1000
     61#define EXPEDITE_THRESHOLD 2000
     62/*
     63 * Max number of callbacks to execute in one go with preemption
     64 * enabled. If there are more callbacks to be executed they will
     65 * be run with preemption disabled in order to prolong reclaimer's
     66 * time slice and give it a chance to catch up with callback producers.
     67 */
     68#define CRITICAL_THRESHOLD 30000
    6269/* Half the number of values a uint32 can hold. */
    6370#define UINT32_MAX_HALF    2147483648U
    6471
    6572
    66 /*c Global RCU data. */
     73/** Global RCU data. */
    6774typedef struct rcu_data {
    6875        /** Detector uses so signal reclaimers that a grace period ended. */
    … …  
    102109        rcu_gp_t completed_gp;
    103110       
    104         /** Protect the following 3 fields. */
     111        /** Protects the following 3 fields. */
    105112        IRQ_SPINLOCK_DECLARE(preempt_lock);
    106113        /** Preexisting readers that have been preempted. */
    … …  
    121128        atomic_t delaying_cpu_cnt;
    122129       
    123         /** Interruptable attached detector thread pointer. */
     130        /** Interruptible attached detector thread pointer. */
    124131        thread_t *detector_thr;
    125132       
    … …  
    153160static void req_detection(size_t req_cnt);
    154161static bool wait_for_cur_cbs_gp_end(bool expedite, rcu_gp_t *last_completed_gp);
     162static void upd_missed_gp_in_wait(rcu_gp_t completed_gp);
    155163static bool cv_wait_for_gp(rcu_gp_t wait_on_gp);
    156164static void detector(void *);
    … …  
    165173static bool wait_for_delaying_cpus(void);
    166174static bool wait_for_preempt_reader(void);
     175static void upd_max_cbs_in_slice(void);
    167176
    168177
    … …  
    210219       
    211220        CPU->rcu.cur_cbs = 0;
     221        CPU->rcu.cur_cbs_cnt = 0;
    212222        CPU->rcu.next_cbs = 0;
     223        CPU->rcu.next_cbs_cnt = 0;
    213224        CPU->rcu.arriving_cbs = 0;
    214225        CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs;
    215        
    216226        CPU->rcu.arriving_cbs_cnt = 0;
    217227
    … …  
    222232       
    223233        semaphore_initialize(&CPU->rcu.arrived_flag, 0);
    224         CPU->rcu.reclaimer_thr = 0;
     234
     235        /* BSP creates reclaimer threads before AP's rcu_cpu_init() runs. */
     236        if (config.cpu_active == 1)
     237                CPU->rcu.reclaimer_thr = 0;
    225238       
    226239        CPU->rcu.stat_max_cbs = 0;
    227240        CPU->rcu.stat_avg_cbs = 0;
    228241        CPU->rcu.stat_missed_gps = 0;
     242        CPU->rcu.stat_missed_gp_in_wait = 0;
     243        CPU->rcu.stat_max_slice_cbs = 0;
     244        CPU->rcu.last_arriving_cnt = 0;
    229245}
    230246
    … …  
    611627{
    612628        ASSERT(THREAD && THREAD->wired);
     629        ASSERT(THREAD == CPU->rcu.reclaimer_thr);
    613630
    614631        rcu_gp_t last_compl_gp = 0;
    … …  
    616633       
    617634        while (ok && wait_for_pending_cbs()) {
     635                ASSERT(CPU->rcu.reclaimer_thr == THREAD);
     636               
    618637                exec_completed_cbs(last_compl_gp);
    619                
     638
    620639                bool expedite = advance_cbs();
    621640               
    … …  
    651670        upd_stat_missed_gp(last_completed_gp);
    652671       
    653         if (CPU->rcu.cur_cbs_gp <= last_completed_gp) {
    654                 exec_cbs(&CPU->rcu.cur_cbs);
    655         }
    656        
     672        /* Both next_cbs and cur_cbs GP elapsed. */
    657673        if (CPU->rcu.next_cbs_gp <= last_completed_gp) {
    658                 exec_cbs(&CPU->rcu.next_cbs);   
     674                ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp);
     675               
     676                size_t exec_cnt = CPU->rcu.cur_cbs_cnt + CPU->rcu.next_cbs_cnt;
     677               
     678                if (exec_cnt < CRITICAL_THRESHOLD) {
     679                        exec_cbs(&CPU->rcu.cur_cbs);
     680                        exec_cbs(&CPU->rcu.next_cbs);   
     681                } else {
     682                        /*
     683                         * Getting overwhelmed with too many callbacks to run.
     684                         * Disable preemption in order to prolong our time slice
     685                         * and catch up with updaters posting new callbacks.
     686                         */
     687                        preemption_disable();
     688                        exec_cbs(&CPU->rcu.cur_cbs);
     689                        exec_cbs(&CPU->rcu.next_cbs);   
     690                        preemption_enable();
     691                }
     692               
     693                CPU->rcu.cur_cbs_cnt = 0;
     694                CPU->rcu.next_cbs_cnt = 0;
     695        } else if (CPU->rcu.cur_cbs_gp <= last_completed_gp) {
     696
     697                if (CPU->rcu.cur_cbs_cnt < CRITICAL_THRESHOLD) {
     698                        exec_cbs(&CPU->rcu.cur_cbs);
     699                } else {
     700                        /*
     701                         * Getting overwhelmed with too many callbacks to run.
     702                         * Disable preemption in order to prolong our time slice
     703                         * and catch up with updaters posting new callbacks.
     704                         */
     705                        preemption_disable();
     706                        exec_cbs(&CPU->rcu.cur_cbs);
     707                        preemption_enable();
     708                }
     709
     710                CPU->rcu.cur_cbs_cnt = 0;
    659711        }
    660712}
    … …  
    696748        /* Move next_cbs to cur_cbs. */
    697749        CPU->rcu.cur_cbs = CPU->rcu.next_cbs;
     750        CPU->rcu.cur_cbs_cnt = CPU->rcu.next_cbs_cnt;
    698751        CPU->rcu.cur_cbs_gp = CPU->rcu.next_cbs_gp;
    699752       
    … …  
    708761                || CPU->rcu.expedite_arriving; 
    709762
    710         /* Update statistics. */
    711         upd_stat_cb_cnts(CPU->rcu.arriving_cbs_cnt);
    712                
    713763        CPU->rcu.expedite_arriving = false;
     764       
    714765        CPU->rcu.next_cbs = CPU->rcu.arriving_cbs;
     766        CPU->rcu.next_cbs_cnt = CPU->rcu.arriving_cbs_cnt;
     767       
    715768        CPU->rcu.arriving_cbs = 0;
    716769        CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs;
    … …  
    718771       
    719772        interrupts_restore(ipl);
     773
     774        /* Update statistics of arrived callbacks. */
     775        upd_stat_cb_cnts(CPU->rcu.next_cbs_cnt);
    720776       
    721777        /*
    … …  
    824880        /* Wait for cur_cbs_gp to end. */
    825881        bool interrupted = cv_wait_for_gp(CPU->rcu.cur_cbs_gp);
    826 
     882       
    827883        *completed_gp = rcu.completed_gp;
    828884        spinlock_unlock(&rcu.gp_lock); 
    829885       
     886        upd_missed_gp_in_wait(*completed_gp);
     887       
    830888        return !interrupted;
    831889}
     890
     891static void upd_missed_gp_in_wait(rcu_gp_t completed_gp)
     892{
     893        ASSERT(CPU->rcu.cur_cbs_gp <= completed_gp);
     894       
     895        size_t delta = (size_t)(completed_gp - CPU->rcu.cur_cbs_gp);
     896        CPU->rcu.stat_missed_gp_in_wait += delta;
     897}
     898
    832899
    833900/** Requests the detector to detect at least req_cnt consecutive grace periods.*/
    … …  
    838905                rcu.req_gp_end_cnt = req_cnt;
    839906
    840                 //printf("reqs:%d,idle:%d ", req_cnt, detector_idle);
    841                
    842907                if (detector_idle) {
    843908                        ASSERT(rcu.cur_gp == rcu.completed_gp);
    844909                        condvar_signal(&rcu.req_gp_changed);
    845910                }
    846         } else {
    847                 //printf("myreqs:%d,detr:%d ", req_cnt, rcu.req_gp_end_cnt);
    848911        }
    849912}
    … …  
    12661329                THREAD->priority = -1;
    12671330        }
     1331        else if (THREAD == CPU->rcu.reclaimer_thr) {
     1332                THREAD->priority = -1;
     1333        }
     1334       
     1335        upd_max_cbs_in_slice();
     1336}
     1337
     1338static void upd_max_cbs_in_slice(void)
     1339{
     1340        rcu_cpu_data_t *cr = &CPU->rcu;
     1341       
     1342        if (cr->arriving_cbs_cnt > cr->last_arriving_cnt) {
     1343                size_t arrived_cnt = cr->arriving_cbs_cnt - cr->last_arriving_cnt;
     1344                cr->stat_max_slice_cbs = max(arrived_cnt, cr->stat_max_slice_cbs);
     1345        }
     1346       
     1347        cr->last_arriving_cnt = cr->arriving_cbs_cnt;
    12681348}
    12691349
    … …  
    12811361void rcu_print_stat(void)
    12821362{
    1283         /* Don't take locks. Worst case is we get out-dated values. */
    1284         printf("Configuration: expedite_threshold=%d, detect_sleep=%dms\n",
    1285                 EXPEDITE_THRESHOLD, DETECT_SLEEP_MS);
     1363        /*
     1364         * Don't take locks. Worst case is we get out-dated values.
     1365         * CPU local values are updated without any locks, so there
     1366         * are no locks to lock in order to get up-to-date values.
     1367         */
     1368       
     1369        printf("Configuration: expedite_threshold=%d, critical_threshold=%d,"
     1370                " detect_sleep=%dms\n",
     1371                EXPEDITE_THRESHOLD, CRITICAL_THRESHOLD, DETECT_SLEEP_MS);
    12861372        printf("Completed GPs: %" PRIu64 "\n", rcu.completed_gp);
    12871373        printf("Expedited GPs: %zu\n", rcu.stat_expedited_cnt);
    … …  
    12921378        printf("Smp calls:     %zu\n", rcu.stat_smp_call_cnt);
    12931379       
    1294         printf("Max callbacks per GP:\n");
    1295         for (unsigned i = 0; i < config.cpu_count; ++i) {
     1380        printf("Max arrived callbacks per GP and CPU:\n");
     1381        for (unsigned int i = 0; i < config.cpu_count; ++i) {
    12961382                printf(" %zu", cpus[i].rcu.stat_max_cbs);
    12971383        }
    12981384
    1299         printf("\nAvg callbacks per GP (nonempty batches only):\n");
    1300         for (unsigned i = 0; i < config.cpu_count; ++i) {
     1385        printf("\nAvg arrived callbacks per GP and CPU (nonempty batches only):\n");
     1386        for (unsigned int i = 0; i < config.cpu_count; ++i) {
    13011387                printf(" %zu", cpus[i].rcu.stat_avg_cbs);
    13021388        }
    13031389       
    1304         printf("\nMissed GP notifications:\n");
    1305         for (unsigned i = 0; i < config.cpu_count; ++i) {
     1390        printf("\nMax arrived callbacks per time slice and CPU:\n");
     1391        for (unsigned int i = 0; i < config.cpu_count; ++i) {
     1392                printf(" %zu", cpus[i].rcu.stat_max_slice_cbs);
     1393        }
     1394
     1395        printf("\nMissed GP notifications per CPU:\n");
     1396        for (unsigned int i = 0; i < config.cpu_count; ++i) {
    13061397                printf(" %zu", cpus[i].rcu.stat_missed_gps);
    13071398        }
     1399
     1400        printf("\nMissed GP notifications per CPU while waking up:\n");
     1401        for (unsigned int i = 0; i < config.cpu_count; ++i) {
     1402                printf(" %zu", cpus[i].rcu.stat_missed_gp_in_wait);
     1403        }
    13081404        printf("\n");
    13091405}
Note: See TracChangeset for help on using the changeset viewer.