X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fbuffer.c;h=5370d50129345949df206454a5a18d30d2c19835;hb=7c03ed4;hp=711b83514f4f5e858cf25533409ba88be0d7f7df;hpb=68b4da67deb2e8ca224bb5abaeb9dbc7ae8e378c;p=vpp.git diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 711b83514f4..5370d501293 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -47,10 +47,17 @@ #include vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0; -static u32 vlib_buffer_physmem_sz = 32 << 20; + +/* when running unpriviledged we are limited by RLIMIT_MEMLOCK which is + typically set to 16MB so setting default size for buffer memory to 14MB + */ +static u32 vlib_buffer_physmem_sz = 14 << 20; vlib_buffer_main_t buffer_main; +/* logging */ +static vlib_log_class_t buffer_log_default; + uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) @@ -379,6 +386,8 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, clib_memset (f, 0, sizeof (f[0])); f->index = f - vm->buffer_free_list_pool; + vec_validate (f->buffers, 0); + vec_reset_length (f->buffers); f->n_data_bytes = vlib_buffer_round_size (n_data_bytes); f->min_n_buffers_each_alloc = VLIB_FRAME_SIZE; f->buffer_pool_index = 0; @@ -405,6 +414,8 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, wf - wvm->buffer_free_list_pool); wf[0] = f[0]; wf->buffers = 0; + vec_validate (wf->buffers, 0); + vec_reset_length (wf->buffers); wf->n_alloc = 0; } @@ -650,26 +661,6 @@ vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers, 0); } -/* Copy template packet data into buffers as they are allocated. */ -static void __attribute__ ((unused)) -vlib_packet_template_buffer_init (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, - u32 * buffers, u32 n_buffers) -{ - vlib_packet_template_t *t = - uword_to_pointer (fl->buffer_init_function_opaque, - vlib_packet_template_t *); - uword i; - - for (i = 0; i < n_buffers; i++) - { - vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); - ASSERT (b->current_length == vec_len (t->packet_data)); - clib_memcpy (vlib_buffer_get_current (b), t->packet_data, - b->current_length); - } -} - void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, @@ -677,20 +668,12 @@ vlib_packet_template_init (vlib_main_t * vm, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt, ...) { - vlib_buffer_main_t *bm = &buffer_main; va_list va; - u8 *name; - vlib_buffer_free_list_t *fl; va_start (va, fmt); - name = va_format (0, fmt, &va); + t->name = va_format (0, fmt, &va); va_end (va); - if (bm->cb.vlib_packet_template_init_cb) - bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data, - n_packet_data_bytes, - min_n_buffers_each_alloc, name); - vlib_worker_thread_barrier_sync (vm); clib_memset (t, 0, sizeof (t[0])); @@ -698,23 +681,6 @@ vlib_packet_template_init (vlib_main_t * vm, vec_add (t->packet_data, packet_data, n_packet_data_bytes); t->min_n_buffers_each_alloc = min_n_buffers_each_alloc; - t->free_list_index = vlib_buffer_create_free_list_helper - (vm, n_packet_data_bytes, - /* is_public */ 1, - /* is_default */ 0, - name); - - ASSERT (t->free_list_index != 0); - fl = vlib_buffer_get_free_list (vm, t->free_list_index); - fl->min_n_buffers_each_alloc = t->min_n_buffers_each_alloc; - - fl->buffer_init_function = vlib_packet_template_buffer_init; - fl->buffer_init_function_opaque = pointer_to_uword (t); - - fl->buffer_init_template.current_data = 0; - fl->buffer_init_template.current_length = n_packet_data_bytes; - fl->buffer_init_template.flags = 0; - fl->buffer_init_template.n_add_refs = 0; vlib_worker_thread_barrier_release (vm); } @@ -731,41 +697,24 @@ vlib_packet_template_get_packet (vlib_main_t * vm, *bi_result = bi; b = vlib_get_buffer (vm, bi); - clib_memcpy (vlib_buffer_get_current (b), - t->packet_data, vec_len (t->packet_data)); + clib_memcpy_fast (vlib_buffer_get_current (b), + t->packet_data, vec_len (t->packet_data)); b->current_length = vec_len (t->packet_data); return b->data; } -void -vlib_packet_template_get_packet_helper (vlib_main_t * vm, - vlib_packet_template_t * t) -{ - word n = t->min_n_buffers_each_alloc; - word l = vec_len (t->packet_data); - word n_alloc; - - ASSERT (l > 0); - ASSERT (vec_len (t->free_buffers) == 0); - - vec_validate (t->free_buffers, n - 1); - n_alloc = vlib_buffer_alloc_from_free_list (vm, t->free_buffers, - n, t->free_list_index); - _vec_len (t->free_buffers) = n_alloc; -} - /* Append given data to end of buffer, possibly allocating new buffers. */ -u32 +int vlib_buffer_add_data (vlib_main_t * vm, vlib_buffer_free_list_index_t free_list_index, - u32 buffer_index, void *data, u32 n_data_bytes) + u32 * buffer_index, void *data, u32 n_data_bytes) { u32 n_buffer_bytes, n_left, n_left_this_buffer, bi; vlib_buffer_t *b; void *d; - bi = buffer_index; + bi = *buffer_index; if (bi == ~0 && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index)) goto out_of_buffers; @@ -789,7 +738,8 @@ vlib_buffer_add_data (vlib_main_t * vm, n_left_this_buffer = n_buffer_bytes - (b->current_data + b->current_length); n = clib_min (n_left_this_buffer, n_left); - clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n); + clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d, + n); b->current_length += n; n_left -= n; if (n_left == 0) @@ -806,11 +756,12 @@ vlib_buffer_add_data (vlib_main_t * vm, b = vlib_get_buffer (vm, b->next_buffer); } - return bi; + *buffer_index = bi; + return 0; out_of_buffers: - clib_error ("out of buffers"); - return bi; + clib_warning ("out of buffers"); + return 1; } u16 @@ -835,13 +786,13 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, free_list_index)) return copied; - *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer); + *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer); max = n_buffer_bytes - l->current_length - l->current_data; } u16 len = (data_len > max) ? max : data_len; - clib_memcpy (vlib_buffer_get_current (l) + l->current_length, - data + copied, len); + clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length, + data + copied, len); vlib_buffer_chain_increase_length (first, l, len); data_len -= len; copied += len; @@ -856,7 +807,7 @@ vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index) vlib_buffer_pool_t *p; vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); uword start = pointer_to_uword (m->base); - uword size = m->n_pages << m->log2_page_size; + uword size = (uword) m->n_pages << m->log2_page_size; if (bm->buffer_mem_size == 0) { @@ -959,13 +910,16 @@ vlib_buffer_main_init (struct vlib_main_t * vm) clib_error_t *error; u32 physmem_map_index; u8 pool_index; + int log2_page_size = 0; + + buffer_log_default = vlib_log_register_class ("buffer", 0); if (vlib_buffer_callbacks) { /* external plugin has registered own buffer callbacks so we just copy them and quit */ - clib_memcpy (&bm->cb, vlib_buffer_callbacks, - sizeof (vlib_buffer_callbacks_t)); + clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks, + sizeof (vlib_buffer_callbacks_t)); bm->callbacks_registered = 1; return 0; } @@ -977,9 +931,24 @@ vlib_buffer_main_init (struct vlib_main_t * vm) &vlib_buffer_delete_free_list_internal; clib_spinlock_init (&bm->buffer_known_hash_lockp); - if ((error = vlib_physmem_shared_map_create (vm, "buffers", - vlib_buffer_physmem_sz, 1, - &physmem_map_index))) +retry: + error = vlib_physmem_shared_map_create (vm, "buffers", + vlib_buffer_physmem_sz, + log2_page_size, + CLIB_PMALLOC_NUMA_LOCAL, + &physmem_map_index); + + if (error && log2_page_size == 0) + { + vlib_log_warn (buffer_log_default, "%U", format_clib_error, error); + clib_error_free (error); + vlib_log_warn (buffer_log_default, "falling back to non-hugepage " + "backed buffer pool"); + log2_page_size = min_log2 (clib_mem_get_page_size ()); + goto retry; + } + + if (error) return error; pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index);