ip: allocate ip4 mtrie pages in htlb memory

[vpp.git] / src / vnet / ip / ip4_mtrie.c

diff --git a/src/vnet/ip/ip4_mtrie.c b/src/vnet/ip/ip4_mtrie.c
index 258a0f7..b5d0a89 100644 (file)
--- a/src/vnet/ip/ip4_mtrie.c
+++ b/src/vnet/ip/ip4_mtrie.c
@@ -790,6 +790,9 @@ format_ip4_fib_mtrie (u8 * s, va_list * va)
 
 /** Default heap size for the IPv4 mtries */
 #define IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE (32<<20)
+#ifndef MAP_HUGE_SHIFT
+#define MAP_HUGE_SHIFT 26
+#endif
 
 static clib_error_t *
 ip4_mtrie_module_init (vlib_main_t * vm)
@@ -799,9 +802,46 @@ ip4_mtrie_module_init (vlib_main_t * vm)
   clib_error_t *error = NULL;
   uword *old_heap;
 
-  if (0 == im->mtrie_heap_size)
+  if (im->mtrie_heap_size == 0)
     im->mtrie_heap_size = IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE;
-  im->mtrie_mheap = create_mspace (im->mtrie_heap_size, 1 /* locked */ );
+
+again:
+  if (im->mtrie_hugetlb)
+    {
+      void *rv;
+      int mmap_flags, mmap_flags_huge;
+      uword htlb_pagesize = clib_mem_get_default_hugepage_size ();
+      if (htlb_pagesize == 0)
+       {
+         clib_warning ("WARNING: htlb pagesize == 0");
+         im->mtrie_hugetlb = 0;
+         goto again;
+       }
+      /* Round the allocation request to an even number of huge pages */
+      im->mtrie_heap_size = (im->mtrie_heap_size + (htlb_pagesize - 1)) &
+       ~(htlb_pagesize - 1);
+      mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
+      mmap_flags_huge = (mmap_flags | MAP_HUGETLB | MAP_LOCKED |
+                        min_log2 (htlb_pagesize) << MAP_HUGE_SHIFT);
+      rv = mmap (0, im->mtrie_heap_size,
+                PROT_READ | PROT_WRITE, mmap_flags_huge, -1, 0);
+      if (rv == MAP_FAILED)
+       {
+         /* Failure when running as root should be logged... */
+         if (geteuid () == 0)
+           clib_warning ("ip4 mtrie htlb map failed: not enough pages?");
+         im->mtrie_hugetlb = 0;
+         goto again;
+       }
+      if (mlock (rv, im->mtrie_heap_size))
+       clib_warning ("WARNING: couldn't lock mtrie heap at %llx", rv);
+      im->mtrie_mheap = create_mspace_with_base (rv, im->mtrie_heap_size,
+                                                1 /* locked */ );
+    }
+  else
+    {
+      im->mtrie_mheap = create_mspace (im->mtrie_heap_size, 1 /* locked */ );
+    }
 
   /* Burn one ply so index 0 is taken */
   old_heap = clib_mem_set_heap (ip4_main.mtrie_mheap);