u16 __vlib_buffer_external_hdr_size = 0;
+static void
+buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e,
+ u32 index);
+
+static void
+buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
+ u32 index);
+
+static void
+buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index);
+
uword
vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
vlib_buffer_t * b_first)
b->total_length_not_including_first_buffer);
if (b->flags & VLIB_BUFFER_IS_TRACED)
- s = format (s, ", trace 0x%x", b->trace_index);
+ s = format (s, ", trace handle 0x%x", b->trace_handle);
if (a)
s = format (s, "\n%U%v", format_white_space, indent, a);
vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
-/* When dubugging validate that given buffers are either known allocated
+/* When debugging validate that given buffers are either known allocated
or known free. */
void
vlib_buffer_validate_alloc_free (vlib_main_t * vm,
return copied;
}
-clib_error_t *
-vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name,
- u32 data_size, u32 physmem_map_index)
+u8
+vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
+ u32 physmem_map_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_pool_t *bp;
uword start = pointer_to_uword (m->base);
uword size = (uword) m->n_pages << m->log2_page_size;
uword i, j;
- u32 alloc_size, n_alloc_per_page;;
+ u32 alloc_size, n_alloc_per_page;
- vec_validate_aligned (bm->buffer_pools, index, CLIB_CACHE_LINE_BYTES);
- bp = vec_elt_at_index (bm->buffer_pools, index);
-
- if (bp->start)
- return clib_error_return (0, "buffer with index %u already exists",
- index);
+ if (vec_len (bm->buffer_pools) >= 255)
+ return ~0;
- if (index >= 255)
- return clib_error_return (0, "buffer index must be < 255", index);
+ vec_add2_aligned (bm->buffer_pools, bp, 1, CLIB_LOG2_CACHE_LINE_BYTES);
if (bm->buffer_mem_size == 0)
{
n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size;
/* preallocate buffer indices memory */
- vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page,
- CLIB_CACHE_LINE_BYTES);
- vec_reset_length (bp->buffers);
+ bp->n_buffers = m->n_pages * n_alloc_per_page;
+ bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32),
+ CLIB_CACHE_LINE_BYTES);
clib_spinlock_init (&bp->lock);
bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p);
- vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES);
+ bp->buffers[bp->n_avail++] = bi;
+
vlib_get_buffer (vm, bi);
}
- bp->n_buffers = vec_len (bp->buffers);
-
- return 0;
+ return bp->index;
}
static u8 *
/* *INDENT-OFF* */
vec_foreach (bpt, bp->threads)
- cached += vec_len (bpt->cached_buffers);
+ cached += bpt->n_cached;
/* *INDENT-ON* */
s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u",
bp->name, bp->index, bp->numa_node, bp->data_size +
sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size,
- bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached,
- bp->n_buffers - vec_len (bp->buffers) - cached);
+ bp->data_size, bp->n_buffers, bp->n_avail, cached,
+ bp->n_buffers - bp->n_avail - cached);
return s;
}
VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init);
static clib_error_t *
-vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node)
+vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
+ u8 * index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
clib_error_t *error;
}
if (error)
- return error;
+ goto done;
vec_reset_length (name);
name = format (name, "default-numa-%d%c", numa_node, 0);
- return vlib_buffer_pool_create (vm, numa_node, (char *) name,
- vlib_buffer_get_default_data_size (vm),
- physmem_map_index);
+ *index = vlib_buffer_pool_create (vm, (char *) name,
+ vlib_buffer_get_default_data_size (vm),
+ physmem_map_index);
+
+ if (*index == (u8) ~ 0)
+ {
+ error = clib_error_return (0, "maximum number of buffer pools reached");
+ goto done;
+ }
+
+done:
+ vec_free (name);
+ return error;
}
void
u32 cached = 0;
vlib_buffer_pool_thread_t *bpt;
+ clib_spinlock_lock (&bp->lock);
+
/* *INDENT-OFF* */
vec_foreach (bpt, bp->threads)
- cached += vec_len (bpt->cached_buffers);
+ cached += bpt->n_cached;
/* *INDENT-ON* */
+ clib_spinlock_unlock (&bp->lock);
+
return cached;
}
static vlib_buffer_pool_t *
-buffer_get_by_name (vlib_buffer_main_t * bm, char *name)
+buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
{
vlib_buffer_pool_t *bp;
- vec_foreach (bp, bm->buffer_pools)
- {
- if (!strcmp ((char *) bp->name, name))
- return bp;
- }
+ if (!bm->buffer_pools || vec_len (bm->buffer_pools) < index)
+ return 0;
+ bp = vec_elt_at_index (bm->buffer_pools, index);
- return 0;
+ return bp;
}
static void
-buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e)
+buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index)
{
vlib_main_t *vm = vlib_get_main ();
- vlib_buffer_pool_t *bp;
-
- bp = buffer_get_by_name (vm->buffer_main,
- &e->name[sizeof ("/buffer/used/") - 1]);
+ vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
if (!bp)
return;
- e->value = bp->n_buffers - vec_len (bp->buffers) - buffer_get_cached (bp);
+ e->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
}
static void
-buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e)
+buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
+ u32 index)
{
vlib_main_t *vm = vlib_get_main ();
- vlib_buffer_pool_t *bp;
-
- bp = buffer_get_by_name (vm->buffer_main,
- &e->name[sizeof ("/buffer/available/") - 1]);
+ vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
if (!bp)
return;
- e->value = vec_len (bp->buffers);
+ e->value = bp->n_avail;
}
static void
-buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e)
+buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index)
{
vlib_main_t *vm = vlib_get_main ();
- vlib_buffer_pool_t *bp;
-
- bp = buffer_get_by_name (vm->buffer_main,
- &e->name[sizeof ("/buffer/cached/") - 1]);
+ vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
if (!bp)
return;
{
vlib_buffer_main_t *bm;
clib_error_t *err;
- clib_bitmap_t *bmp = 0;
+ clib_bitmap_t *bmp = 0, *bmp_has_memory = 0;
u32 numa_node;
vlib_buffer_pool_t *bp;
- u8 *name;
+ u8 *name = 0, first_valid_buffer_pool_index = ~0;
vlib_buffer_main_alloc (vm);
clib_spinlock_init (&bm->buffer_known_hash_lockp);
- err = clib_sysfs_read ("/sys/devices/system/node/possible", "%U",
- unformat_bitmap_list, &bmp);
- if (err)
+ if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
+ unformat_bitmap_list, &bmp)))
+ clib_error_free (err);
+
+ if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U",
+ unformat_bitmap_list, &bmp_has_memory)))
+ clib_error_free (err);
+
+ if (bmp && bmp_has_memory)
+ bmp = clib_bitmap_and (bmp, bmp_has_memory);
+
+ /* no info from sysfs, assuming that only numa 0 exists */
+ if (bmp == 0)
+ bmp = clib_bitmap_set (bmp, 0, 1);
+
+ if (clib_bitmap_last_set (bmp) >= VLIB_BUFFER_MAX_NUMA_NODES)
+ clib_panic ("system have more than %u NUMA nodes",
+ VLIB_BUFFER_MAX_NUMA_NODES);
+
+ /* *INDENT-OFF* */
+ clib_bitmap_foreach (numa_node, bmp,
+ {
+ u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
+ index[0] = ~0;
+ if ((err = vlib_buffer_main_init_numa_node (vm, numa_node, index)))
+ {
+ clib_error_report (err);
+ clib_error_free (err);
+ continue;
+ }
+
+ if (first_valid_buffer_pool_index == 0xff)
+ first_valid_buffer_pool_index = index[0];
+ });
+ /* *INDENT-ON* */
+
+ if (first_valid_buffer_pool_index == (u8) ~ 0)
{
- /* no info from sysfs, assuming that only numa 0 exists */
- clib_error_free (err);
- bmp = clib_bitmap_set (bmp, 0, 1);
+ err = clib_error_return (0, "failed to allocate buffer pool(s)");
+ goto done;
}
/* *INDENT-OFF* */
- clib_bitmap_foreach (numa_node, bmp, {
- if ((err = vlib_buffer_main_init_numa_node(vm, numa_node)))
- goto done;
+ clib_bitmap_foreach (numa_node, bmp,
+ {
+ if (bm->default_buffer_pool_index_for_numa[numa_node] == (u8) ~0)
+ bm->default_buffer_pool_index_for_numa[numa_node] =
+ first_valid_buffer_pool_index;
});
/* *INDENT-ON* */
- bm->n_numa_nodes = clib_bitmap_last_set (bmp) + 1;
-
vec_foreach (bp, bm->buffer_pools)
{
- name = format (0, "/buffer/cached/%s%c", bp->name, 0);
- stat_segment_register_gauge (name, buffer_gauges_update_cached_fn);
- vec_free (name);
- name = format (0, "/buffer/used/%s%c", bp->name, 0);
- stat_segment_register_gauge (name, buffer_gauges_update_used_fn);
- vec_free (name);
- name = format (0, "/buffer/available/%s%c", bp->name, 0);
- stat_segment_register_gauge (name, buffer_gauges_update_available_fn);
- vec_free (name);
+ if (bp->n_buffers == 0)
+ continue;
+
+ vec_reset_length (name);
+ name = format (name, "/buffer-pools/%s/cached%c", bp->name, 0);
+ stat_segment_register_gauge (name, buffer_gauges_update_cached_fn,
+ bp - bm->buffer_pools);
+
+ vec_reset_length (name);
+ name = format (name, "/buffer-pools/%s/used%c", bp->name, 0);
+ stat_segment_register_gauge (name, buffer_gauges_update_used_fn,
+ bp - bm->buffer_pools);
+
+ vec_reset_length (name);
+ name = format (name, "/buffer-pools/%s/available%c", bp->name, 0);
+ stat_segment_register_gauge (name, buffer_gauges_update_available_fn,
+ bp - bm->buffer_pools);
}
-
done:
vec_free (bmp);
+ vec_free (bmp_has_memory);
+ vec_free (name);
return err;
}