else if (new_leaf_dst_address_bits >=
ply->dst_address_bits_of_leaves[i])
{
- clib_atomic_cmp_and_swap (&ply->leaves[i], old_leaf, new_leaf);
- ASSERT (ply->leaves[i] == new_leaf);
+ clib_atomic_store_rel_n (&ply->leaves[i], new_leaf);
ply->dst_address_bits_of_leaves[i] = new_leaf_dst_address_bits;
ply->n_non_empty_leafs += ip4_fib_mtrie_leaf_is_non_empty (ply, i);
}
old_ply->dst_address_bits_of_leaves[i] =
a->dst_address_length;
- clib_atomic_cmp_and_swap (&old_ply->leaves[i], old_leaf,
- new_leaf);
- ASSERT (old_ply->leaves[i] == new_leaf);
+ clib_atomic_store_rel_n (&old_ply->leaves[i], new_leaf);
old_ply->n_non_empty_leafs +=
ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
/* Refetch since ply_create may move pool. */
old_ply = pool_elt_at_index (ip4_ply_pool, old_ply_index);
- clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
- ASSERT (old_ply->leaves[dst_byte] == new_leaf);
+ clib_atomic_store_rel_n (&old_ply->leaves[dst_byte], new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
old_ply->n_non_empty_leafs +=
* the new one */
old_ply->dst_address_bits_of_leaves[slot] =
a->dst_address_length;
- clib_atomic_cmp_and_swap (&old_ply->leaves[slot],
- old_leaf, new_leaf);
- ASSERT (old_ply->leaves[slot] == new_leaf);
+ clib_atomic_store_rel_n (&old_ply->leaves[slot], new_leaf);
}
else
{
ply_base_len);
new_ply = get_next_ply_for_leaf (m, new_leaf);
- clib_atomic_cmp_and_swap (&old_ply->leaves[dst_byte], old_leaf,
- new_leaf);
- ASSERT (old_ply->leaves[dst_byte] == new_leaf);
+ clib_atomic_store_rel_n (&old_ply->leaves[dst_byte], new_leaf);
old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
}
else
old_ply->n_non_empty_leafs -=
ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
- old_ply->leaves[i] =
- ip4_fib_mtrie_leaf_set_adj_index (a->cover_adj_index);
+ clib_atomic_store_rel_n (&old_ply->leaves[i],
+ ip4_fib_mtrie_leaf_set_adj_index
+ (a->cover_adj_index));
old_ply->dst_address_bits_of_leaves[i] = a->cover_address_length;
old_ply->n_non_empty_leafs +=
|| (!old_leaf_is_terminal
&& unset_leaf (m, a, get_next_ply_for_leaf (m, old_leaf), 2)))
{
- old_ply->leaves[slot] =
- ip4_fib_mtrie_leaf_set_adj_index (a->cover_adj_index);
+ clib_atomic_store_rel_n (&old_ply->leaves[slot],
+ ip4_fib_mtrie_leaf_set_adj_index
+ (a->cover_adj_index));
old_ply->dst_address_bits_of_leaves[slot] = a->cover_address_length;
}
}
/** Default heap size for the IPv4 mtries */
#define IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE (32<<20)
+#ifndef MAP_HUGE_SHIFT
+#define MAP_HUGE_SHIFT 26
+#endif
static clib_error_t *
ip4_mtrie_module_init (vlib_main_t * vm)
clib_error_t *error = NULL;
uword *old_heap;
- if (0 == im->mtrie_heap_size)
+ if (im->mtrie_heap_size == 0)
im->mtrie_heap_size = IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE;
-#if USE_DLMALLOC == 0
- im->mtrie_mheap = mheap_alloc (0, im->mtrie_heap_size);
-#else
- im->mtrie_mheap = create_mspace (im->mtrie_heap_size, 1 /* locked */ );
-#endif
+
+again:
+ if (im->mtrie_hugetlb)
+ {
+ void *rv;
+ int mmap_flags, mmap_flags_huge;
+ uword htlb_pagesize = clib_mem_get_default_hugepage_size ();
+ if (htlb_pagesize == 0)
+ {
+ clib_warning ("WARNING: htlb pagesize == 0");
+ im->mtrie_hugetlb = 0;
+ goto again;
+ }
+ /* Round the allocation request to an even number of huge pages */
+ im->mtrie_heap_size = (im->mtrie_heap_size + (htlb_pagesize - 1)) &
+ ~(htlb_pagesize - 1);
+ mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
+ mmap_flags_huge = (mmap_flags | MAP_HUGETLB | MAP_LOCKED |
+ min_log2 (htlb_pagesize) << MAP_HUGE_SHIFT);
+ rv = mmap (0, im->mtrie_heap_size,
+ PROT_READ | PROT_WRITE, mmap_flags_huge, -1, 0);
+ if (rv == MAP_FAILED)
+ {
+ /* Failure when running as root should be logged... */
+ if (geteuid () == 0)
+ clib_warning ("ip4 mtrie htlb map failed: not enough pages?");
+ im->mtrie_hugetlb = 0;
+ goto again;
+ }
+ if (mlock (rv, im->mtrie_heap_size))
+ clib_warning ("WARNING: couldn't lock mtrie heap at %llx", rv);
+ im->mtrie_mheap = create_mspace_with_base (rv, im->mtrie_heap_size,
+ 1 /* locked */ );
+ }
+ else
+ {
+ im->mtrie_mheap = create_mspace (im->mtrie_heap_size, 1 /* locked */ );
+ }
/* Burn one ply so index 0 is taken */
old_heap = clib_mem_set_heap (ip4_main.mtrie_mheap);