return result;
}
-/*
- * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
- * and vlib_mains[0] = &vlib_global_main from the beginning of time.
- *
- * The only place which should ever expand vlib_mains is start_workers()
- * in threads.c. It knows about the bootstrap vector.
- */
-/* *INDENT-OFF* */
-static struct
-{
- vec_header_t h;
- vlib_main_t *vm;
-} __attribute__ ((packed)) __bootstrap_vlib_main_vector
- __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
-{
- .h.len = 1,
- .vm = &vlib_global_main,
-};
-/* *INDENT-ON* */
-
-vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
-
-
/* When debugging validate that given buffers are either known allocated
or known free. */
void
if (vec_len (bm->buffer_pools) >= 255)
return ~0;
- vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_LOG2_CACHE_LINE_BYTES);
+ vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_CACHE_LINE_BYTES);
if (bm->buffer_mem_size == 0)
{
bp->data_size = data_size;
bp->numa_node = m->numa_node;
- vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1,
+ vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
CLIB_CACHE_LINE_BYTES);
alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size);
return s;
}
-static clib_error_t *
-show_buffers (vlib_main_t * vm,
- unformat_input_t * input, vlib_cli_command_t * cmd)
+u8 *
+format_vlib_buffer_pool_all (u8 *s, va_list *va)
{
+ vlib_main_t *vm = va_arg (*va, vlib_main_t *);
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_pool_t *bp;
- vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, 0);
+ s = format (s, "%U", format_vlib_buffer_pool, vm, 0);
- /* *INDENT-OFF* */
vec_foreach (bp, bm->buffer_pools)
- vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, bp);
- /* *INDENT-ON* */
+ s = format (s, "\n%U", format_vlib_buffer_pool, vm, bp);
+ return s;
+}
+
+static clib_error_t *
+show_buffers (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ vlib_cli_output (vm, "%U", format_vlib_buffer_pool_all, vm);
return 0;
}
vec_foreach (bp, bm->buffer_pools)
{
clib_spinlock_lock (&bp->lock);
- vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1,
+ vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
CLIB_CACHE_LINE_BYTES);
clib_spinlock_unlock (&bp->lock);
}
VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init);
static clib_error_t *
-vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
- u8 * index)
+vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node,
+ u32 * physmem_map_index,
+ clib_mem_page_sz_t log2_page_size,
+ u8 unpriv)
{
vlib_buffer_main_t *bm = vm->buffer_main;
+ u32 buffers_per_numa = bm->buffers_per_numa;
clib_error_t *error;
- u32 physmem_map_index;
+ u32 buffer_size;
uword n_pages, pagesize;
- u32 buffers_per_numa;
- u32 buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size,
- vlib_buffer_get_default_data_size
- (vm));
- u8 *name;
+ u8 *name = 0;
- pagesize = clib_mem_get_default_hugepage_size ();
- name = format (0, "buffers-numa-%d%c", numa_node, 0);
-
- buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa :
- VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
-
-retry:
+ ASSERT (log2_page_size != CLIB_MEM_PAGE_SZ_UNKNOWN);
+ pagesize = clib_mem_page_bytes (log2_page_size);
+ buffer_size = vlib_buffer_alloc_size (bm->ext_hdr_size,
+ vlib_buffer_get_default_data_size
+ (vm));
if (buffer_size > pagesize)
- {
- error =
- clib_error_return (0,
- "buffer size (%llu) is greater than page size (%llu)",
- buffer_size, pagesize);
- goto done;
- }
+ return clib_error_return (0, "buffer size (%llu) is greater than page "
+ "size (%llu)", buffer_size, pagesize);
+
+ if (buffers_per_numa == 0)
+ buffers_per_numa = unpriv ? VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV :
+ VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA;
+ name = format (0, "buffers-numa-%d%c", numa_node, 0);
n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1;
error = vlib_physmem_shared_map_create (vm, (char *) name,
n_pages * pagesize,
min_log2 (pagesize), numa_node,
- &physmem_map_index);
+ physmem_map_index);
+ vec_free (name);
+ return error;
+}
- if (error && pagesize != clib_mem_get_page_size ())
+static clib_error_t *
+vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
+ u8 * index)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ u32 physmem_map_index;
+ clib_error_t *error;
+ u8 *name = 0;
+
+ if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN)
{
- vlib_log_warn (bm->log_default, "%U", format_clib_error, error);
+ error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+ &physmem_map_index,
+ CLIB_MEM_PAGE_SZ_DEFAULT_HUGE,
+ 0 /* unpriv */ );
+ if (!error)
+ goto buffer_pool_create;
+
+ /* If alloc failed, retry without hugepages */
+ vlib_log_warn (bm->log_default,
+ "numa[%u] falling back to non-hugepage backed "
+ "buffer pool (%U)", numa_node, format_clib_error, error);
clib_error_free (error);
- vlib_log_warn (bm->log_default, "falling back to non-hugepage "
- "backed buffer pool");
- pagesize = clib_mem_get_page_size ();
- buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa :
- VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV;
- goto retry;
- }
+ error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+ &physmem_map_index,
+ CLIB_MEM_PAGE_SZ_DEFAULT,
+ 1 /* unpriv */ );
+ }
+ else
+ error = vlib_buffer_main_init_numa_alloc (vm, numa_node,
+ &physmem_map_index,
+ bm->log2_page_size,
+ 0 /* unpriv */ );
if (error)
- goto done;
+ return error;
- vec_reset_length (name);
+buffer_pool_create:
name = format (name, "default-numa-%d%c", numa_node, 0);
-
*index = vlib_buffer_pool_create (vm, (char *) name,
vlib_buffer_get_default_data_size (vm),
physmem_map_index);
if (*index == (u8) ~ 0)
- {
- error = clib_error_return (0, "maximum number of buffer pools reached");
- goto done;
- }
-
-done:
+ error = clib_error_return (0, "maximum number of buffer pools reached");
vec_free (name);
+
+
return error;
}
VLIB_BUFFER_MAX_NUMA_NODES);
/* *INDENT-OFF* */
- clib_bitmap_foreach (numa_node, bmp,
+ clib_bitmap_foreach (numa_node, bmp)
{
u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
index[0] = ~0;
if (first_valid_buffer_pool_index == 0xff)
first_valid_buffer_pool_index = index[0];
- });
+ }
/* *INDENT-ON* */
if (first_valid_buffer_pool_index == (u8) ~ 0)
}
/* *INDENT-OFF* */
- clib_bitmap_foreach (numa_node, bmp,
+ clib_bitmap_foreach (numa_node, bmp)
{
if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0)
bm->default_buffer_pool_index_for_numa[numa_node] =
first_valid_buffer_pool_index;
- });
+ }
/* *INDENT-ON* */
vec_foreach (bp, bm->buffer_pools)
vlib_buffer_main_alloc (vm);
bm = vm->buffer_main;
+ bm->log2_page_size = CLIB_MEM_PAGE_SZ_UNKNOWN;
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa))
;
+ else if (unformat (input, "page-size %U", unformat_log2_page_size,
+ &bm->log2_page_size))
+ ;
else if (unformat (input, "default data-size %u",
&bm->default_data_size))
;
}
#endif
+__clib_export int
+vlib_buffer_set_alloc_free_callback (
+ vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
+ vlib_buffer_alloc_free_callback_t *free_callback_fn)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ if ((alloc_callback_fn && bm->alloc_callback_fn) ||
+ (free_callback_fn && bm->free_callback_fn))
+ return 1;
+ bm->alloc_callback_fn = alloc_callback_fn;
+ bm->free_callback_fn = free_callback_fn;
+ return 0;
+}
+
/** @endcond */
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff