b->next_buffer = t->buffers[i];
b->flags |= VLIB_BUFFER_NEXT_PRESENT;
b = vlib_get_buffer (vm, b->next_buffer);
- clib_memcpy_fast (b, bt, sizeof (vlib_buffer_t));
+ vlib_buffer_copy_template (b, bt);
tlnifb += b->current_length = qw1 >> AVF_RXD_LEN_SHIFT;
i++;
}
avf_per_thread_data_t * ptd, u32 n_left,
int maybe_multiseg)
{
- vlib_buffer_t *bt = &ptd->buffer_template;
+ vlib_buffer_t bt;
vlib_buffer_t **b = ptd->bufs;
u64 *qw1 = ptd->qw1s;
avf_rx_tail_t *tail = ptd->tails;
uword n_rx_bytes = 0;
+ /* copy template into local variable - will save per packet load */
+ vlib_buffer_copy_template (&bt, &ptd->buffer_template);
+
while (n_left >= 4)
{
if (n_left >= 12)
vlib_prefetch_buffer_header (b[11], LOAD);
}
- clib_memcpy64_x4 (b[0], b[1], b[2], b[3], bt);
+ vlib_buffer_copy_template (b[0], &bt);
+ vlib_buffer_copy_template (b[1], &bt);
+ vlib_buffer_copy_template (b[2], &bt);
+ vlib_buffer_copy_template (b[3], &bt);
n_rx_bytes += b[0]->current_length = qw1[0] >> AVF_RXD_LEN_SHIFT;
n_rx_bytes += b[1]->current_length = qw1[1] >> AVF_RXD_LEN_SHIFT;
if (maybe_multiseg)
{
- n_rx_bytes += avf_rx_attach_tail (vm, bt, b[0], qw1[0], tail + 0);
- n_rx_bytes += avf_rx_attach_tail (vm, bt, b[1], qw1[1], tail + 1);
- n_rx_bytes += avf_rx_attach_tail (vm, bt, b[2], qw1[2], tail + 2);
- n_rx_bytes += avf_rx_attach_tail (vm, bt, b[3], qw1[3], tail + 3);
+ n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[0], qw1[0], tail + 0);
+ n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[1], qw1[1], tail + 1);
+ n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[2], qw1[2], tail + 2);
+ n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[3], qw1[3], tail + 3);
}
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
}
while (n_left)
{
- clib_memcpy_fast (b[0], bt, sizeof (vlib_buffer_t));
+ vlib_buffer_copy_template (b[0], &bt);
n_rx_bytes += b[0]->current_length = qw1[0] >> AVF_RXD_LEN_SHIFT;
if (maybe_multiseg)
- n_rx_bytes += avf_rx_attach_tail (vm, bt, b[0], qw1[0], tail + 0);
+ n_rx_bytes += avf_rx_attach_tail (vm, &bt, b[0], qw1[0], tail + 0);
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
no_prefetch:
vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8,
sizeof (struct rte_mbuf));
- clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[0]),
- vlib_buffer_from_rte_mbuf (mb[1]),
- vlib_buffer_from_rte_mbuf (mb[2]),
- vlib_buffer_from_rte_mbuf (mb[3]), &bt);
- clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[4]),
- vlib_buffer_from_rte_mbuf (mb[5]),
- vlib_buffer_from_rte_mbuf (mb[6]),
- vlib_buffer_from_rte_mbuf (mb[7]), &bt);
+
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[0]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[1]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[2]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[3]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[4]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[5]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[6]), &bt);
+ vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[7]), &bt);
n_left -= 8;
mb += 8;
static_always_inline uword
dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b,
- struct rte_mbuf *mb, vlib_buffer_free_list_t * fl)
+ struct rte_mbuf *mb, vlib_buffer_t * bt)
{
u8 nb_seg = 1;
struct rte_mbuf *mb_seg = 0;
ASSERT (mb_seg != 0);
b_seg = vlib_buffer_from_rte_mbuf (mb_seg);
- vlib_buffer_init_for_free_list (b_seg, fl);
-
- ASSERT ((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
- ASSERT (b_seg->current_data == 0);
+ vlib_buffer_copy_template (b_seg, bt);
/*
* The driver (e.g. virtio) may not put the packet data at the start
{
u32 n_left = n_rx_packets;
vlib_buffer_t *b[4];
- vlib_buffer_free_list_t *fl;
struct rte_mbuf **mb = ptd->mbufs;
uword n_bytes = 0;
u8 *flags, or_flags = 0;
-
- if (maybe_multiseg)
- fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
+ vlib_buffer_t bt;
mb = ptd->mbufs;
flags = ptd->flags;
+ /* copy template into local variable - will save per packet load */
+ vlib_buffer_copy_template (&bt, &ptd->buffer_template);
while (n_left >= 8)
{
dpdk_prefetch_buffer_x4 (mb + 4);
b[2] = vlib_buffer_from_rte_mbuf (mb[2]);
b[3] = vlib_buffer_from_rte_mbuf (mb[3]);
- clib_memcpy64_x4 (b[0], b[1], b[2], b[3], &ptd->buffer_template);
+ vlib_buffer_copy_template (b[0], &bt);
+ vlib_buffer_copy_template (b[1], &bt);
+ vlib_buffer_copy_template (b[2], &bt);
+ vlib_buffer_copy_template (b[3], &bt);
dpdk_prefetch_mbuf_x4 (mb + 4);
if (maybe_multiseg)
{
- n_bytes += dpdk_process_subseq_segs (vm, b[0], mb[0], fl);
- n_bytes += dpdk_process_subseq_segs (vm, b[1], mb[1], fl);
- n_bytes += dpdk_process_subseq_segs (vm, b[2], mb[2], fl);
- n_bytes += dpdk_process_subseq_segs (vm, b[3], mb[3], fl);
+ n_bytes += dpdk_process_subseq_segs (vm, b[0], mb[0], &bt);
+ n_bytes += dpdk_process_subseq_segs (vm, b[1], mb[1], &bt);
+ n_bytes += dpdk_process_subseq_segs (vm, b[2], mb[2], &bt);
+ n_bytes += dpdk_process_subseq_segs (vm, b[3], mb[3], &bt);
}
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
while (n_left)
{
b[0] = vlib_buffer_from_rte_mbuf (mb[0]);
- clib_memcpy_fast (b[0], &ptd->buffer_template, 64);
+ vlib_buffer_copy_template (b[0], &bt);
or_flags |= dpdk_ol_flags_extract (mb, flags, 1);
flags += 1;
n_bytes += b[0]->current_length = mb[0]->data_len;
if (maybe_multiseg)
- n_bytes += dpdk_process_subseq_segs (vm, b[0], mb[0], fl);
+ n_bytes += dpdk_process_subseq_segs (vm, b[0], mb[0], &bt);
VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]);
/* next */
if (mm->per_thread_data == 0)
{
int i;
- vlib_buffer_free_list_t *fl;
vec_validate_aligned (mm->per_thread_data, tm->n_vlib_mains - 1,
CLIB_CACHE_LINE_BYTES);
- fl =
- vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
for (i = 0; i < tm->n_vlib_mains; i++)
{
memif_per_thread_data_t *ptd =
vec_elt_at_index (mm->per_thread_data, i);
vlib_buffer_t *bt = &ptd->buffer_template;
- vlib_buffer_init_for_free_list (bt, fl);
+ clib_memset (bt, 0, sizeof (vlib_buffer_t));
bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
bt->total_length_not_including_first_buffer = 0;
vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0;
memif_main_t *mm = &memif_main;
memif_ring_t *ring;
memif_queue_t *mq;
- u16 buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
+ u16 buffer_size = VLIB_BUFFER_DATA_SIZE;
uword n_trace = vlib_get_trace_count (vm, node);
u16 nexts[MEMIF_RX_VECTOR_SZ], *next = nexts;
u32 _to_next_bufs[MEMIF_RX_VECTOR_SZ], *to_next_bufs = _to_next_bufs, *bi;
u32 thread_index = vm->thread_index;
memif_per_thread_data_t *ptd = vec_elt_at_index (mm->per_thread_data,
thread_index);
- vlib_buffer_t *bt = &ptd->buffer_template;
+ vlib_buffer_t bt;
u16 cur_slot, last_slot, ring_size, n_slots, mask;
i16 start_offset;
u16 n_buffers = 0, n_alloc;
}
/* prepare buffer template and next indices */
- vnet_buffer (bt)->sw_if_index[VLIB_RX] = mif->sw_if_index;
- vnet_buffer (bt)->feature_arc_index = 0;
- bt->current_data = start_offset;
- bt->current_config_index = 0;
+ vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_RX] =
+ mif->sw_if_index;
+ vnet_buffer (&ptd->buffer_template)->feature_arc_index = 0;
+ ptd->buffer_template.current_data = start_offset;
+ ptd->buffer_template.current_config_index = 0;
if (mode == MEMIF_INTERFACE_MODE_ETHERNET)
{
next_index = mif->per_interface_next_index;
else
vnet_feature_start_device_input_x1 (mif->sw_if_index, &next_index,
- bt);
+ &ptd->buffer_template);
vlib_get_new_next_frame (vm, node, next_index, to_next_bufs,
n_left_to_next);
po = ptd->packet_ops;
bi = to_next_bufs;
+ /* copy template into local variable - will save per packet load */
+ vlib_buffer_copy_template (&bt, &ptd->buffer_template);
+
while (n_from >= 8)
{
b0 = vlib_get_buffer (vm, po[4].first_buffer_vec_index);
b2 = vlib_get_buffer (vm, bi[2]);
b3 = vlib_get_buffer (vm, bi[3]);
- clib_memcpy64_x4 (b0, b1, b2, b3, bt);
+ vlib_buffer_copy_template (b0, &bt);
+ vlib_buffer_copy_template (b1, &bt);
+ vlib_buffer_copy_template (b2, &bt);
+ vlib_buffer_copy_template (b3, &bt);
b0->current_length = po[0].packet_len;
n_rx_bytes += b0->current_length;
fbvi[0] = po[0].first_buffer_vec_index;
bi[0] = ptd->buffers[fbvi[0]];
b0 = vlib_get_buffer (vm, bi[0]);
- clib_memcpy_fast (b0, bt, 64);
+ vlib_buffer_copy_template (b0, &bt);
b0->current_length = po->packet_len;
n_rx_bytes += b0->current_length;
/* asume that somebody will want to add ethernet header on the packet
so start with IP header at offset 14 */
start_offset = (mode == MEMIF_INTERFACE_MODE_IP) ? 14 : 0;
- buffer_length = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES - start_offset;
+ buffer_length = VLIB_BUFFER_DATA_SIZE - start_offset;
cur_slot = mq->last_tail;
last_slot = ring->tail;
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- STRUCT_MARK (template_start);
/* Offset within data[] that we are currently processing.
If negative current header points into predata area. */
i16 current_data; /**< signed offset in data[], pre_data[]
Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
*/
- STRUCT_MARK (template_end);
-
u32 current_config_index; /**< Used by feature subgraph arcs to
visit enabled feature nodes
*/
u32 opaque[10]; /**< Opaque data used by sub-graphs for their own purposes.
See .../vnet/vnet/buffer.h
*/
+
+ STRUCT_MARK (template_end); /**< part of buffer metadata which is
+ initialized on alloc ends here. It may be
+ different than cacheline on systems with
+ buffer cacheline size */
+
+ /***** end of first cache line */
CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
u32 trace_index; /**< Specifies index into trace buffer
return uword_to_pointer (bm->buffer_mem_start + offset, void *);
}
+static_always_inline void
+vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt)
+{
+ clib_memcpy_fast (b, bt, STRUCT_OFFSET_OF (vlib_buffer_t, template_end));
+}
+
/** \brief Translate array of buffer indices into buffer pointers with offset
@param vm - (vlib_main_t *) vlib main data structure pointer
CLIB_CACHE_LINE_BYTES * 2);
/* Make sure buffer template is sane. */
- ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
-
- clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+ vlib_buffer_copy_template (dst, src);
/* Not in the first 16 octets. */
dst->n_add_refs = src->n_add_refs;
*/
#define clib_memset(s,c,n) memset_s_inline(s,n,c,n)
-/*
- * Copy 64 bytes of data to 4 destinations
- * this function is typically used in quad-loop case when whole cacheline
- * needs to be copied to 4 different places. First it reads whole cacheline
- * to 1/2/4 SIMD registers and then it writes data to 4 destinations.
- */
-
-static_always_inline void
-clib_memcpy64_x4 (void *d0, void *d1, void *d2, void *d3, void *s)
-{
-#if defined (__AVX512F__)
- __m512i r0 = _mm512_loadu_si512 (s);
-
- _mm512_storeu_si512 (d0, r0);
- _mm512_storeu_si512 (d1, r0);
- _mm512_storeu_si512 (d2, r0);
- _mm512_storeu_si512 (d3, r0);
-
-#elif defined (__AVX2__)
- __m256i r0 = _mm256_loadu_si256 ((__m256i *) (s + 0 * 32));
- __m256i r1 = _mm256_loadu_si256 ((__m256i *) (s + 1 * 32));
-
- _mm256_storeu_si256 ((__m256i *) (d0 + 0 * 32), r0);
- _mm256_storeu_si256 ((__m256i *) (d0 + 1 * 32), r1);
-
- _mm256_storeu_si256 ((__m256i *) (d1 + 0 * 32), r0);
- _mm256_storeu_si256 ((__m256i *) (d1 + 1 * 32), r1);
-
- _mm256_storeu_si256 ((__m256i *) (d2 + 0 * 32), r0);
- _mm256_storeu_si256 ((__m256i *) (d2 + 1 * 32), r1);
-
- _mm256_storeu_si256 ((__m256i *) (d3 + 0 * 32), r0);
- _mm256_storeu_si256 ((__m256i *) (d3 + 1 * 32), r1);
-
-#elif defined (__SSSE3__)
- __m128i r0 = _mm_loadu_si128 ((__m128i *) (s + 0 * 16));
- __m128i r1 = _mm_loadu_si128 ((__m128i *) (s + 1 * 16));
- __m128i r2 = _mm_loadu_si128 ((__m128i *) (s + 2 * 16));
- __m128i r3 = _mm_loadu_si128 ((__m128i *) (s + 3 * 16));
-
- _mm_storeu_si128 ((__m128i *) (d0 + 0 * 16), r0);
- _mm_storeu_si128 ((__m128i *) (d0 + 1 * 16), r1);
- _mm_storeu_si128 ((__m128i *) (d0 + 2 * 16), r2);
- _mm_storeu_si128 ((__m128i *) (d0 + 3 * 16), r3);
-
- _mm_storeu_si128 ((__m128i *) (d1 + 0 * 16), r0);
- _mm_storeu_si128 ((__m128i *) (d1 + 1 * 16), r1);
- _mm_storeu_si128 ((__m128i *) (d1 + 2 * 16), r2);
- _mm_storeu_si128 ((__m128i *) (d1 + 3 * 16), r3);
-
- _mm_storeu_si128 ((__m128i *) (d2 + 0 * 16), r0);
- _mm_storeu_si128 ((__m128i *) (d2 + 1 * 16), r1);
- _mm_storeu_si128 ((__m128i *) (d2 + 2 * 16), r2);
- _mm_storeu_si128 ((__m128i *) (d2 + 3 * 16), r3);
-
- _mm_storeu_si128 ((__m128i *) (d3 + 0 * 16), r0);
- _mm_storeu_si128 ((__m128i *) (d3 + 1 * 16), r1);
- _mm_storeu_si128 ((__m128i *) (d3 + 2 * 16), r2);
- _mm_storeu_si128 ((__m128i *) (d3 + 3 * 16), r3);
-
-#else
- clib_memcpy_fast (d0, s, 64);
- clib_memcpy_fast (d1, s, 64);
- clib_memcpy_fast (d2, s, 64);
- clib_memcpy_fast (d3, s, 64);
-#endif
-}
-
static_always_inline void
clib_memset_u64 (void *p, u64 val, uword count)
{
Code Review / vpp.git / commitdiff