Changeset 63e27ef in mainline for kernel/generic/src/adt/cht.c
- Timestamp:
- 2017-06-19T21:47:42Z (7 years ago)
- Branches:
- lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
- Children:
- deacc58d
- Parents:
- 7354b5e
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
kernel/generic/src/adt/cht.c
r7354b5e r63e27ef 290 290 #include <adt/cht.h> 291 291 #include <adt/hash.h> 292 #include < debug.h>292 #include <assert.h> 293 293 #include <mm/slab.h> 294 294 #include <arch/barrier.h> … … 522 522 bool can_block, cht_ops_t *op) 523 523 { 524 ASSERT(h);525 ASSERT(op && op->hash && op->key_hash && op->equal && op->key_equal);524 assert(h); 525 assert(op && op->hash && op->key_hash && op->equal && op->key_equal); 526 526 /* Memoized hashes are stored in the rcu_link.func function pointer. */ 527 STATIC_ASSERT(sizeof(size_t) == sizeof(rcu_func_t));528 ASSERT(sentinel.hash == (uintptr_t)sentinel.rcu_link.func);527 static_assert(sizeof(size_t) == sizeof(rcu_func_t), ""); 528 assert(sentinel.hash == (uintptr_t)sentinel.rcu_link.func); 529 529 530 530 /* All operations are compulsory. */ … … 625 625 626 626 /* You must clear the table of items. Otherwise cht_destroy will leak. */ 627 ASSERT(atomic_get(&h->item_cnt) == 0);627 assert(atomic_get(&h->item_cnt) == 0); 628 628 } 629 629 … … 685 685 static inline cht_link_t *find_lazy(cht_t *h, void *key) 686 686 { 687 ASSERT(h);687 assert(h); 688 688 /* See docs to cht_find() and cht_find_lazy(). */ 689 ASSERT(rcu_read_locked());689 assert(rcu_read_locked()); 690 690 691 691 size_t hash = calc_key_hash(h, key); … … 731 731 cht_link_t *cht_find_next_lazy(cht_t *h, const cht_link_t *item) 732 732 { 733 ASSERT(h);734 ASSERT(rcu_read_locked());735 ASSERT(item);733 assert(h); 734 assert(rcu_read_locked()); 735 assert(item); 736 736 737 737 return find_duplicate(h, item, calc_node_hash(h, item), get_next(item->link)); … … 755 755 do { 756 756 cur = get_next(prev); 757 ASSERT(cur);757 assert(cur); 758 758 prev = cur->link; 759 759 } while (node_hash(h, cur) < search_hash); … … 770 770 771 771 cur = get_next(cur->link); 772 ASSERT(cur);772 assert(cur); 773 773 } 774 774 … … 790 790 marked_ptr_t old_head, size_t old_idx) 791 791 { 792 ASSERT(N_INVALID == get_mark(old_head));793 ASSERT(h->new_b);792 assert(N_INVALID == get_mark(old_head)); 793 assert(h->new_b); 794 794 795 795 size_t new_idx = calc_bucket_idx(hash, h->new_b->order); … … 903 903 * traversing search_head. 904 904 */ 905 ASSERT(N_JOIN & get_mark(get_next(old_head)->link));905 assert(N_JOIN & get_mark(get_next(old_head)->link)); 906 906 return search_bucket(h, old_head, key, hash); 907 907 } … … 913 913 * sure all cpus see that the new table replaced the old one. 914 914 */ 915 ASSERT(h->b->order == h->new_b->order);915 assert(h->b->order == h->new_b->order); 916 916 /* 917 917 * The resizer must ensure all new bucket heads are visible before 918 918 * replacing the old table. 919 919 */ 920 ASSERT(N_NORMAL == get_mark(new_head));920 assert(N_NORMAL == get_mark(new_head)); 921 921 return search_bucket(h, new_head, key, hash); 922 922 } … … 961 961 bool cht_insert_unique(cht_t *h, cht_link_t *item, cht_link_t **dup_item) 962 962 { 963 ASSERT(rcu_read_locked());964 ASSERT(dup_item);963 assert(rcu_read_locked()); 964 assert(dup_item); 965 965 return insert_impl(h, item, dup_item); 966 966 } … … 1060 1060 return ret == make_link(wnd->cur, jf_mark); 1061 1061 } else { 1062 ASSERT(walk_mode == WM_LEAVE_JOIN);1062 assert(walk_mode == WM_LEAVE_JOIN); 1063 1063 1064 1064 item->link = make_link(wnd->cur, N_NORMAL); … … 1087 1087 cht_link_t *cur, cht_link_t **dup_item) 1088 1088 { 1089 ASSERT(cur);1090 ASSERT(cur == &sentinel || hash <= node_hash(h, cur)1089 assert(cur); 1090 assert(cur == &sentinel || hash <= node_hash(h, cur) 1091 1091 || node_hash(h, cur) == h->invalid_hash); 1092 1092 … … 1110 1110 cht_link_t *start) 1111 1111 { 1112 ASSERT(hash <= node_hash(h, start) || h->invalid_hash == node_hash(h, start));1112 assert(hash <= node_hash(h, start) || h->invalid_hash == node_hash(h, start)); 1113 1113 1114 1114 cht_link_t *cur = start; 1115 1115 1116 1116 try_again: 1117 ASSERT(cur);1117 assert(cur); 1118 1118 1119 1119 while (node_hash(h, cur) == hash) { 1120 ASSERT(cur != &sentinel);1120 assert(cur != &sentinel); 1121 1121 1122 1122 bool deleted = (N_DELETED & get_mark(cur->link)); … … 1127 1127 1128 1128 cur = get_next(cur->link); 1129 ASSERT(cur);1129 assert(cur); 1130 1130 } 1131 1131 … … 1142 1142 size_t cht_remove_key(cht_t *h, void *key) 1143 1143 { 1144 ASSERT(h);1144 assert(h); 1145 1145 1146 1146 size_t hash = calc_key_hash(h, key); … … 1163 1163 bool cht_remove_item(cht_t *h, cht_link_t *item) 1164 1164 { 1165 ASSERT(h);1166 ASSERT(item);1165 assert(h); 1166 assert(item); 1167 1167 /* Otherwise a concurrent cht_remove_key might free the item. */ 1168 ASSERT(rcu_read_locked());1168 assert(rcu_read_locked()); 1169 1169 1170 1170 /* … … 1262 1262 bool *deleted_but_gc, bool *resizing) 1263 1263 { 1264 ASSERT(wnd->cur && wnd->cur != &sentinel);1264 assert(wnd->cur && wnd->cur != &sentinel); 1265 1265 1266 1266 *deleted_but_gc = false; … … 1292 1292 bool *resizing) 1293 1293 { 1294 ASSERT(cur && cur != &sentinel);1294 assert(cur && cur != &sentinel); 1295 1295 1296 1296 /* … … 1310 1310 } 1311 1311 } else { 1312 STATIC_ASSERT(N_JOIN == N_JOIN_FOLLOWS);1312 static_assert(N_JOIN == N_JOIN_FOLLOWS, ""); 1313 1313 1314 1314 /* Keep the N_JOIN/N_JOIN_FOLLOWS mark but strip N_DELETED. */ … … 1329 1329 bool *resizing) 1330 1330 { 1331 ASSERT(wnd->cur != &sentinel);1332 ASSERT(wnd->cur && (N_DELETED & get_mark(wnd->cur->link)));1331 assert(wnd->cur != &sentinel); 1332 assert(wnd->cur && (N_DELETED & get_mark(wnd->cur->link))); 1333 1333 1334 1334 cht_link_t *next = get_next(wnd->cur->link); … … 1336 1336 if (walk_mode == WM_LEAVE_JOIN) { 1337 1337 /* Never try to unlink join nodes. */ 1338 ASSERT(!(N_JOIN & get_mark(wnd->cur->link)));1338 assert(!(N_JOIN & get_mark(wnd->cur->link))); 1339 1339 1340 1340 mark_t pred_mark = get_mark(*wnd->ppred); … … 1348 1348 return false; 1349 1349 } else { 1350 ASSERT(walk_mode == WM_MOVE_JOIN_FOLLOWS || walk_mode == WM_NORMAL);1350 assert(walk_mode == WM_MOVE_JOIN_FOLLOWS || walk_mode == WM_NORMAL); 1351 1351 /* Move the JF mark if set. Clear DEL mark. */ 1352 1352 mark_t cur_mark = N_JOIN_FOLLOWS & get_mark(wnd->cur->link); … … 1398 1398 equal_pred_t pred, void *pred_arg, wnd_t *wnd, bool *resizing) 1399 1399 { 1400 ASSERT(wnd->cur);1400 assert(wnd->cur); 1401 1401 1402 1402 if (wnd->cur == &sentinel) … … 1415 1415 1416 1416 while (cur_hash <= hash) { 1417 ASSERT(wnd->cur && wnd->cur != &sentinel);1417 assert(wnd->cur && wnd->cur != &sentinel); 1418 1418 1419 1419 /* GC any deleted nodes on the way. */ … … 1436 1436 if (cur_hash == h->invalid_hash) { 1437 1437 next_wnd(wnd); 1438 ASSERT(wnd->cur);1438 assert(wnd->cur); 1439 1439 goto try_again; 1440 1440 } … … 1469 1469 { 1470 1470 try_again: 1471 ASSERT(wnd->cur);1471 assert(wnd->cur); 1472 1472 1473 1473 while (node_hash(h, wnd->cur) < hash) { … … 1482 1482 } 1483 1483 1484 ASSERT(wnd->cur);1484 assert(wnd->cur); 1485 1485 } 1486 1486 … … 1498 1498 bool *resizing) 1499 1499 { 1500 ASSERT(N_DELETED & get_mark(wnd->cur->link));1500 assert(N_DELETED & get_mark(wnd->cur->link)); 1501 1501 1502 1502 /* Skip deleted JOIN nodes. */ … … 1505 1505 } else { 1506 1506 /* Ordinary deleted node or a deleted JOIN_FOLLOWS. */ 1507 ASSERT(walk_mode != WM_LEAVE_JOIN1507 assert(walk_mode != WM_LEAVE_JOIN 1508 1508 || !((N_JOIN | N_JOIN_FOLLOWS) & get_mark(wnd->cur->link))); 1509 1509 … … 1546 1546 * it 1547 1547 */ 1548 ASSERT(h->b->order > h->new_b->order);1549 ASSERT(wnd->cur);1548 assert(h->b->order > h->new_b->order); 1549 assert(wnd->cur); 1550 1550 1551 1551 /* Either we did not need the joining link or we have already followed it.*/ … … 1620 1620 /* The hash belongs to the moved bucket. */ 1621 1621 if (move_dest_idx == new_idx) { 1622 ASSERT(pmoved_head == pnew_head);1622 assert(pmoved_head == pnew_head); 1623 1623 /* 1624 1624 * move_head() makes the new head of the moved bucket visible. 1625 1625 * The new head may be marked with a JOIN_FOLLOWS 1626 1626 */ 1627 ASSERT(!(N_CONST & get_mark(*pmoved_head)));1627 assert(!(N_CONST & get_mark(*pmoved_head))); 1628 1628 *walk_mode = WM_MOVE_JOIN_FOLLOWS; 1629 1629 } else { 1630 ASSERT(pmoved_head != pnew_head);1630 assert(pmoved_head != pnew_head); 1631 1631 /* 1632 1632 * The hash belongs to the bucket that is the result of splitting … … 1643 1643 * JOIN_FOLLOWS in this part of split bucket. 1644 1644 */ 1645 ASSERT(N_NORMAL == get_mark(*pnew_head));1645 assert(N_NORMAL == get_mark(*pnew_head)); 1646 1646 } 1647 1647 … … 1675 1675 1676 1676 /* move_head() or join_buckets() makes it so or makes the mark visible.*/ 1677 ASSERT(N_INVALID == get_mark(*pold_head));1677 assert(N_INVALID == get_mark(*pold_head)); 1678 1678 /* move_head() makes it visible. No JOIN_FOLLOWS used when shrinking. */ 1679 ASSERT(N_NORMAL == get_mark(*pnew_head));1679 assert(N_NORMAL == get_mark(*pnew_head)); 1680 1680 1681 1681 *walk_mode = WM_LEAVE_JOIN; … … 1685 1685 * readers to notice that the old table had been replaced. 1686 1686 */ 1687 ASSERT(b == h->new_b);1687 assert(b == h->new_b); 1688 1688 *walk_mode = WM_NORMAL; 1689 1689 } … … 1712 1712 { 1713 1713 /* Head move has to in progress already when calling this func. */ 1714 ASSERT(N_CONST & get_mark(*psrc_head));1714 assert(N_CONST & get_mark(*psrc_head)); 1715 1715 1716 1716 /* Head already moved. */ … … 1725 1725 } 1726 1726 1727 ASSERT(!(N_CONST & get_mark(*pdest_head)));1727 assert(!(N_CONST & get_mark(*pdest_head))); 1728 1728 } 1729 1729 … … 1756 1756 static void complete_head_move(marked_ptr_t *psrc_head, marked_ptr_t *pdest_head) 1757 1757 { 1758 ASSERT(N_JOIN_FOLLOWS != get_mark(*psrc_head));1759 ASSERT(N_CONST & get_mark(*psrc_head));1758 assert(N_JOIN_FOLLOWS != get_mark(*psrc_head)); 1759 assert(N_CONST & get_mark(*psrc_head)); 1760 1760 1761 1761 cht_link_t *next = get_next(*psrc_head); … … 1763 1763 DBG(marked_ptr_t ret = ) 1764 1764 cas_link(pdest_head, &sentinel, N_INVALID, next, N_NORMAL); 1765 ASSERT(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret)));1765 assert(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret))); 1766 1766 cas_order_barrier(); 1767 1767 1768 1768 DBG(ret = ) 1769 1769 cas_link(psrc_head, next, N_CONST, next, N_INVALID); 1770 ASSERT(ret == make_link(next, N_CONST) || (N_INVALID == get_mark(ret)));1770 assert(ret == make_link(next, N_CONST) || (N_INVALID == get_mark(ret))); 1771 1771 cas_order_barrier(); 1772 1772 } … … 1861 1861 DBG(marked_ptr_t ret = ) 1862 1862 cas_link(pdest_head, &sentinel, N_INVALID, wnd.cur, N_NORMAL); 1863 ASSERT(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret)));1863 assert(ret == make_link(&sentinel, N_INVALID) || (N_NORMAL == get_mark(ret))); 1864 1864 cas_order_barrier(); 1865 1865 … … 1905 1905 1906 1906 /* Must not report that the table is resizing if WM_MOVE_JOIN_FOLLOWS.*/ 1907 ASSERT(!resizing);1907 assert(!resizing); 1908 1908 /* 1909 1909 * Mark the last node of the first half of the split bucket … … 2042 2042 DBG(marked_ptr_t ret = ) 2043 2043 cas_link(psrc_head, join_node, N_CONST, join_node, N_INVALID); 2044 ASSERT(ret == make_link(join_node, N_CONST) || (N_INVALID == get_mark(ret)));2044 assert(ret == make_link(join_node, N_CONST) || (N_INVALID == get_mark(ret))); 2045 2045 cas_order_barrier(); 2046 2046 … … 2073 2073 continue; 2074 2074 2075 ASSERT(!resizing);2075 assert(!resizing); 2076 2076 2077 2077 if (wnd.cur != &sentinel) { 2078 2078 /* Must be from the new appended bucket. */ 2079 ASSERT(split_hash <= node_hash(h, wnd.cur)2079 assert(split_hash <= node_hash(h, wnd.cur) 2080 2080 || h->invalid_hash == node_hash(h, wnd.cur)); 2081 2081 return; … … 2096 2096 static void free_later(cht_t *h, cht_link_t *item) 2097 2097 { 2098 ASSERT(item != &sentinel);2098 assert(item != &sentinel); 2099 2099 2100 2100 /* … … 2156 2156 2157 2157 #ifdef CONFIG_DEBUG 2158 ASSERT(h->b);2158 assert(h->b); 2159 2159 /* Make resize_reqs visible. */ 2160 2160 read_barrier(); 2161 ASSERT(0 < atomic_get(&h->resize_reqs));2161 assert(0 < atomic_get(&h->resize_reqs)); 2162 2162 #endif 2163 2163 … … 2336 2336 /* Set the invalid joinee head to NULL. */ 2337 2337 if (old_idx != move_src_idx) { 2338 ASSERT(N_INVALID == get_mark(h->b->head[old_idx]));2338 assert(N_INVALID == get_mark(h->b->head[old_idx])); 2339 2339 2340 2340 if (&sentinel != get_next(h->b->head[old_idx])) … … 2392 2392 marked_ptr_t *new_head) 2393 2393 { 2394 ASSERT(join_node != &sentinel);2395 ASSERT(join_node && (N_JOIN & get_mark(join_node->link)));2394 assert(join_node != &sentinel); 2395 assert(join_node && (N_JOIN & get_mark(join_node->link))); 2396 2396 2397 2397 bool done; … … 2409 2409 /* Done if the mark was cleared. Retry if a new node was inserted. */ 2410 2410 done = (ret == jn_link); 2411 ASSERT(ret == jn_link || (get_mark(ret) & N_JOIN));2411 assert(ret == jn_link || (get_mark(ret) & N_JOIN)); 2412 2412 } while (!done); 2413 2413 … … 2432 2432 join_node, &wnd, &resizing); 2433 2433 2434 ASSERT(!resizing);2434 assert(!resizing); 2435 2435 } while (!done); 2436 2436 } … … 2439 2439 static void cleanup_join_follows(cht_t *h, marked_ptr_t *new_head) 2440 2440 { 2441 ASSERT(new_head);2441 assert(new_head); 2442 2442 2443 2443 rcu_read_lock(); … … 2464 2464 /* GC any deleted nodes on the way - even deleted JOIN_FOLLOWS. */ 2465 2465 if (N_DELETED & get_mark(*cur_link)) { 2466 ASSERT(cur_link != new_head);2467 ASSERT(wnd.ppred && wnd.cur && wnd.cur != &sentinel);2468 ASSERT(cur_link == &wnd.cur->link);2466 assert(cur_link != new_head); 2467 assert(wnd.ppred && wnd.cur && wnd.cur != &sentinel); 2468 assert(cur_link == &wnd.cur->link); 2469 2469 2470 2470 bool dummy; … … 2484 2484 cas_link(cur_link, next, N_JOIN_FOLLOWS, &sentinel, N_NORMAL); 2485 2485 2486 ASSERT(next == &sentinel2486 assert(next == &sentinel 2487 2487 || ((N_JOIN | N_JOIN_FOLLOWS) & get_mark(ret))); 2488 2488 … … 2505 2505 2506 2506 /* We must encounter a JF node before we reach the end of the bucket. */ 2507 ASSERT(wnd.cur && wnd.cur != &sentinel);2507 assert(wnd.cur && wnd.cur != &sentinel); 2508 2508 cur_link = &wnd.cur->link; 2509 2509 } … … 2519 2519 static inline size_t calc_split_hash(size_t split_idx, size_t order) 2520 2520 { 2521 ASSERT(1 <= order && order <= 8 * sizeof(size_t));2521 assert(1 <= order && order <= 8 * sizeof(size_t)); 2522 2522 return split_idx << (8 * sizeof(size_t) - order); 2523 2523 } … … 2526 2526 static inline size_t calc_bucket_idx(size_t hash, size_t order) 2527 2527 { 2528 ASSERT(1 <= order && order <= 8 * sizeof(size_t));2528 assert(1 <= order && order <= 8 * sizeof(size_t)); 2529 2529 return hash >> (8 * sizeof(size_t) - order); 2530 2530 } … … 2558 2558 static inline size_t node_hash(cht_t *h, const cht_link_t *item) 2559 2559 { 2560 ASSERT(item->hash == h->invalid_hash2560 assert(item->hash == h->invalid_hash 2561 2561 || item->hash == sentinel.hash 2562 2562 || item->hash == calc_node_hash(h, item)); … … 2568 2568 static inline size_t calc_node_hash(cht_t *h, const cht_link_t *item) 2569 2569 { 2570 ASSERT(item != &sentinel);2570 assert(item != &sentinel); 2571 2571 /* 2572 2572 * Clear the lowest order bit in order for sentinel's node hash … … 2587 2587 marked_ptr_t ptr = (marked_ptr_t) next; 2588 2588 2589 ASSERT(!(ptr & N_MARK_MASK));2590 ASSERT(!((unsigned)mark & ~N_MARK_MASK));2589 assert(!(ptr & N_MARK_MASK)); 2590 assert(!((unsigned)mark & ~N_MARK_MASK)); 2591 2591 2592 2592 return ptr | mark; … … 2608 2608 static inline void next_wnd(wnd_t *wnd) 2609 2609 { 2610 ASSERT(wnd);2611 ASSERT(wnd->cur);2610 assert(wnd); 2611 assert(wnd->cur); 2612 2612 2613 2613 wnd->last = wnd->cur; … … 2635 2635 marked_ptr_t new) 2636 2636 { 2637 ASSERT(link != &sentinel.link);2637 assert(link != &sentinel.link); 2638 2638 /* 2639 2639 * cas(x) on the same location x on one cpu must be ordered, but do not
Note:
See TracChangeset
for help on using the changeset viewer.