Index: kernel/generic/src/synch/rcu.c
===================================================================
--- kernel/generic/src/synch/rcu.c	(revision c14762e308251eb9ee22f1638076409ea9196fc9)
+++ kernel/generic/src/synch/rcu.c	(revision 1c1da4bb5da22780fc373a9c9faa4e7fdc641791)
@@ -54,15 +54,22 @@
  * when non-expedited grace period detection is in progress.
  */
-#define DETECT_SLEEP_MS    5
+#define DETECT_SLEEP_MS    10
 /* 
  * Max number of pending callbacks in the local cpu's queue before 
  * aggressively expediting the current grace period
  */
-#define EXPEDITE_THRESHOLD 1000
+#define EXPEDITE_THRESHOLD 2000
+/*
+ * Max number of callbacks to execute in one go with preemption
+ * enabled. If there are more callbacks to be executed they will
+ * be run with preemption disabled in order to prolong reclaimer's
+ * time slice and give it a chance to catch up with callback producers.
+ */
+#define CRITICAL_THRESHOLD 30000
 /* Half the number of values a uint32 can hold. */
 #define UINT32_MAX_HALF    2147483648U
 
 
-/*c Global RCU data. */
+/** Global RCU data. */
 typedef struct rcu_data {
 	/** Detector uses so signal reclaimers that a grace period ended. */
@@ -102,5 +109,5 @@
 	rcu_gp_t completed_gp;
 	
-	/** Protect the following 3 fields. */
+	/** Protects the following 3 fields. */
 	IRQ_SPINLOCK_DECLARE(preempt_lock);
 	/** Preexisting readers that have been preempted. */
@@ -121,5 +128,5 @@
 	atomic_t delaying_cpu_cnt;
 	
-	/** Interruptable attached detector thread pointer. */
+	/** Interruptible attached detector thread pointer. */
 	thread_t *detector_thr;
 	
@@ -153,4 +160,5 @@
 static void req_detection(size_t req_cnt);
 static bool wait_for_cur_cbs_gp_end(bool expedite, rcu_gp_t *last_completed_gp);
+static void upd_missed_gp_in_wait(rcu_gp_t completed_gp);
 static bool cv_wait_for_gp(rcu_gp_t wait_on_gp);
 static void detector(void *);
@@ -165,4 +173,5 @@
 static bool wait_for_delaying_cpus(void);
 static bool wait_for_preempt_reader(void);
+static void upd_max_cbs_in_slice(void);
 
 
@@ -210,8 +219,9 @@
 	
 	CPU->rcu.cur_cbs = 0;
+	CPU->rcu.cur_cbs_cnt = 0;
 	CPU->rcu.next_cbs = 0;
+	CPU->rcu.next_cbs_cnt = 0;
 	CPU->rcu.arriving_cbs = 0;
 	CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs;
-	
 	CPU->rcu.arriving_cbs_cnt = 0;
 
@@ -222,9 +232,15 @@
 	
 	semaphore_initialize(&CPU->rcu.arrived_flag, 0);
-	CPU->rcu.reclaimer_thr = 0;
+
+	/* BSP creates reclaimer threads before AP's rcu_cpu_init() runs. */
+	if (config.cpu_active == 1)
+		CPU->rcu.reclaimer_thr = 0;
 	
 	CPU->rcu.stat_max_cbs = 0;
 	CPU->rcu.stat_avg_cbs = 0;
 	CPU->rcu.stat_missed_gps = 0;
+	CPU->rcu.stat_missed_gp_in_wait = 0;
+	CPU->rcu.stat_max_slice_cbs = 0;
+	CPU->rcu.last_arriving_cnt = 0;
 }
 
@@ -611,4 +627,5 @@
 {
 	ASSERT(THREAD && THREAD->wired);
+	ASSERT(THREAD == CPU->rcu.reclaimer_thr);
 
 	rcu_gp_t last_compl_gp = 0;
@@ -616,6 +633,8 @@
 	
 	while (ok && wait_for_pending_cbs()) {
+		ASSERT(CPU->rcu.reclaimer_thr == THREAD);
+		
 		exec_completed_cbs(last_compl_gp);
-		
+
 		bool expedite = advance_cbs();
 		
@@ -651,10 +670,43 @@
 	upd_stat_missed_gp(last_completed_gp);
 	
-	if (CPU->rcu.cur_cbs_gp <= last_completed_gp) {
-		exec_cbs(&CPU->rcu.cur_cbs);
-	}
-	
+	/* Both next_cbs and cur_cbs GP elapsed. */
 	if (CPU->rcu.next_cbs_gp <= last_completed_gp) {
-		exec_cbs(&CPU->rcu.next_cbs);	
+		ASSERT(CPU->rcu.cur_cbs_gp <= CPU->rcu.next_cbs_gp);
+		
+		size_t exec_cnt = CPU->rcu.cur_cbs_cnt + CPU->rcu.next_cbs_cnt;
+		
+		if (exec_cnt < CRITICAL_THRESHOLD) {
+			exec_cbs(&CPU->rcu.cur_cbs);
+			exec_cbs(&CPU->rcu.next_cbs);	
+		} else {
+			/* 
+			 * Getting overwhelmed with too many callbacks to run. 
+			 * Disable preemption in order to prolong our time slice 
+			 * and catch up with updaters posting new callbacks.
+			 */
+			preemption_disable();
+			exec_cbs(&CPU->rcu.cur_cbs);
+			exec_cbs(&CPU->rcu.next_cbs);	
+			preemption_enable();
+		}
+		
+		CPU->rcu.cur_cbs_cnt = 0;
+		CPU->rcu.next_cbs_cnt = 0;
+	} else if (CPU->rcu.cur_cbs_gp <= last_completed_gp) {
+
+		if (CPU->rcu.cur_cbs_cnt < CRITICAL_THRESHOLD) {
+			exec_cbs(&CPU->rcu.cur_cbs);
+		} else {
+			/* 
+			 * Getting overwhelmed with too many callbacks to run. 
+			 * Disable preemption in order to prolong our time slice 
+			 * and catch up with updaters posting new callbacks.
+			 */
+			preemption_disable();
+			exec_cbs(&CPU->rcu.cur_cbs);
+			preemption_enable();
+		}
+
+		CPU->rcu.cur_cbs_cnt = 0;
 	}
 }
@@ -696,4 +748,5 @@
 	/* Move next_cbs to cur_cbs. */
 	CPU->rcu.cur_cbs = CPU->rcu.next_cbs;
+	CPU->rcu.cur_cbs_cnt = CPU->rcu.next_cbs_cnt;
 	CPU->rcu.cur_cbs_gp = CPU->rcu.next_cbs_gp;
 	
@@ -708,9 +761,9 @@
 		|| CPU->rcu.expedite_arriving;	
 
-	/* Update statistics. */
-	upd_stat_cb_cnts(CPU->rcu.arriving_cbs_cnt);
-		
 	CPU->rcu.expedite_arriving = false;
+	
 	CPU->rcu.next_cbs = CPU->rcu.arriving_cbs;
+	CPU->rcu.next_cbs_cnt = CPU->rcu.arriving_cbs_cnt;
+	
 	CPU->rcu.arriving_cbs = 0;
 	CPU->rcu.parriving_cbs_tail = &CPU->rcu.arriving_cbs;
@@ -718,4 +771,7 @@
 	
 	interrupts_restore(ipl);
+
+	/* Update statistics of arrived callbacks. */
+	upd_stat_cb_cnts(CPU->rcu.next_cbs_cnt);
 	
 	/* 
@@ -824,10 +880,21 @@
 	/* Wait for cur_cbs_gp to end. */
 	bool interrupted = cv_wait_for_gp(CPU->rcu.cur_cbs_gp);
-
+	
 	*completed_gp = rcu.completed_gp;
 	spinlock_unlock(&rcu.gp_lock);	
 	
+	upd_missed_gp_in_wait(*completed_gp);
+	
 	return !interrupted;
 }
+
+static void upd_missed_gp_in_wait(rcu_gp_t completed_gp)
+{
+	ASSERT(CPU->rcu.cur_cbs_gp <= completed_gp);
+	
+	size_t delta = (size_t)(completed_gp - CPU->rcu.cur_cbs_gp);
+	CPU->rcu.stat_missed_gp_in_wait += delta;
+}
+
 
 /** Requests the detector to detect at least req_cnt consecutive grace periods.*/
@@ -838,12 +905,8 @@
 		rcu.req_gp_end_cnt = req_cnt;
 
-		//printf("reqs:%d,idle:%d ", req_cnt, detector_idle);
-		
 		if (detector_idle) {
 			ASSERT(rcu.cur_gp == rcu.completed_gp);
 			condvar_signal(&rcu.req_gp_changed);
 		}
-	} else {
-		//printf("myreqs:%d,detr:%d ", req_cnt, rcu.req_gp_end_cnt);
 	}
 }
@@ -1266,4 +1329,21 @@
 		THREAD->priority = -1;
 	} 
+	else if (THREAD == CPU->rcu.reclaimer_thr) {
+		THREAD->priority = -1;
+	} 
+	
+	upd_max_cbs_in_slice();
+}
+
+static void upd_max_cbs_in_slice(void)
+{
+	rcu_cpu_data_t *cr = &CPU->rcu;
+	
+	if (cr->arriving_cbs_cnt > cr->last_arriving_cnt) {
+		size_t arrived_cnt = cr->arriving_cbs_cnt - cr->last_arriving_cnt;
+		cr->stat_max_slice_cbs = max(arrived_cnt, cr->stat_max_slice_cbs);
+	}
+	
+	cr->last_arriving_cnt = cr->arriving_cbs_cnt;
 }
 
@@ -1281,7 +1361,13 @@
 void rcu_print_stat(void)
 {
-	/* Don't take locks. Worst case is we get out-dated values. */
-	printf("Configuration: expedite_threshold=%d, detect_sleep=%dms\n",
-		EXPEDITE_THRESHOLD, DETECT_SLEEP_MS);
+	/* 
+	 * Don't take locks. Worst case is we get out-dated values. 
+	 * CPU local values are updated without any locks, so there 
+	 * are no locks to lock in order to get up-to-date values.
+	 */
+	
+	printf("Configuration: expedite_threshold=%d, critical_threshold=%d,"
+		" detect_sleep=%dms\n",	
+		EXPEDITE_THRESHOLD, CRITICAL_THRESHOLD, DETECT_SLEEP_MS);
 	printf("Completed GPs: %" PRIu64 "\n", rcu.completed_gp);
 	printf("Expedited GPs: %zu\n", rcu.stat_expedited_cnt);
@@ -1292,18 +1378,28 @@
 	printf("Smp calls:     %zu\n", rcu.stat_smp_call_cnt);
 	
-	printf("Max callbacks per GP:\n");
-	for (unsigned i = 0; i < config.cpu_count; ++i) {
+	printf("Max arrived callbacks per GP and CPU:\n");
+	for (unsigned int i = 0; i < config.cpu_count; ++i) {
 		printf(" %zu", cpus[i].rcu.stat_max_cbs);
 	}
 
-	printf("\nAvg callbacks per GP (nonempty batches only):\n");
-	for (unsigned i = 0; i < config.cpu_count; ++i) {
+	printf("\nAvg arrived callbacks per GP and CPU (nonempty batches only):\n");
+	for (unsigned int i = 0; i < config.cpu_count; ++i) {
 		printf(" %zu", cpus[i].rcu.stat_avg_cbs);
 	}
 	
-	printf("\nMissed GP notifications:\n");
-	for (unsigned i = 0; i < config.cpu_count; ++i) {
+	printf("\nMax arrived callbacks per time slice and CPU:\n");
+	for (unsigned int i = 0; i < config.cpu_count; ++i) {
+		printf(" %zu", cpus[i].rcu.stat_max_slice_cbs);
+	}
+
+	printf("\nMissed GP notifications per CPU:\n");
+	for (unsigned int i = 0; i < config.cpu_count; ++i) {
 		printf(" %zu", cpus[i].rcu.stat_missed_gps);
 	}
+
+	printf("\nMissed GP notifications per CPU while waking up:\n");
+	for (unsigned int i = 0; i < config.cpu_count; ++i) {
+		printf(" %zu", cpus[i].rcu.stat_missed_gp_in_wait);
+	}
 	printf("\n");
 }