#define included_vlib_buffer_funcs_h
#include <vppinfra/hash.h>
+#include <vppinfra/fifo.h>
/** \file
vlib buffer access methods.
u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
/* shift and add to get vlib_buffer_t pointer */
u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
+#elif defined (CLIB_HAVE_VEC128)
+ u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset);
+ u32x4 bi4 = u32x4_load_unaligned (bi);
+ u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
+#if defined (__aarch64__)
+ u64x2 b1 = u32x4_extend_to_u64x2_high ((u32x4) bi4);
+#else
+ bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
+ u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
+#endif
+ u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
+ u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
#else
b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset;
b[1] = ((u8 *) vlib_get_buffer (vm, bi[1])) + offset;
}
while (count)
{
- bi[0] = vlib_get_buffer_index (vm, b[0] + offset);
+ bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
bi += 1;
b += 1;
count -= 1;
{
b = vlib_get_buffer (vm, buffer_index);
l = b->current_length;
- clib_memcpy (contents + content_len, b->data + b->current_data, l);
+ clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
content_len += l;
if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
break;
return content_len;
}
-/* Return physical address of buffer->data start. */
-always_inline u64
-vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index)
+always_inline uword
+vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b)
{
- vlib_buffer_main_t *bm = &buffer_main;
- vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index);
- vlib_buffer_pool_t *pool = vec_elt_at_index (bm->buffer_pools,
- b->buffer_pool_index);
+ return vlib_physmem_get_pa (vm, b->data);
+}
- return vlib_physmem_virtual_to_physical (vm, pool->physmem_region, b->data);
+always_inline uword
+vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b)
+{
+ return vlib_buffer_get_pa (vm, b) + b->current_data;
}
/** \brief Prefetch buffer metadata by buffer index
vlib_prefetch_buffer_header (_b, type); \
} while (0)
-#if 0
-/* Iterate over known allocated vlib bufs. You probably do not want
- * to do this!
- @param vm the vlib_main_t
- @param bi found allocated buffer index
- @param body operation to perform on buffer index
- function executes body for each allocated buffer index
- */
-#define vlib_buffer_foreach_allocated(vm,bi,body) \
-do { \
- vlib_main_t * _vmain = (vm); \
- vlib_buffer_main_t * _bmain = &_vmain->buffer_main; \
- hash_pair_t * _vbpair; \
- hash_foreach_pair(_vbpair, _bmain->buffer_known_hash, ({ \
- if (VLIB_BUFFER_KNOWN_ALLOCATED == _vbpair->value[0]) { \
- (bi) = _vbpair->key; \
- body; \
- } \
- })); \
-} while (0)
-#endif
-
typedef enum
{
/* Index is unknown. */
/* following code is intentionaly duplicated to allow compiler
to optimize fast path when n_buffers is constant value */
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
}
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
if (PREDICT_TRUE (start + n_buffers <= ring_size))
{
- vlib_buffer_free (vm, ring + start, n_buffers);
+ vlib_buffer_free_no_next (vm, ring + start, n_buffers);
}
else
{
return f->n_data_bytes;
}
-void vlib_aligned_memcpy (void *_dst, void *_src, int n_bytes);
-
-/* Reasonably fast buffer copy routine. */
-always_inline void
-vlib_copy_buffers (u32 * dst, u32 * src, u32 n)
-{
- while (n >= 4)
- {
- dst[0] = src[0];
- dst[1] = src[1];
- dst[2] = src[2];
- dst[3] = src[3];
- dst += 4;
- src += 4;
- n -= 4;
- }
- while (n > 0)
- {
- dst[0] = src[0];
- dst += 1;
- src += 1;
- n -= 1;
- }
-}
-
/* Append given data to end of buffer, possibly allocating new buffers. */
u32 vlib_buffer_add_data (vlib_main_t * vm,
vlib_buffer_free_list_index_t free_list_index,
d->flags = s->flags & flag_mask;
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
/* next segments */
for (i = 1; i < n_buffers; i++)
d = vlib_get_buffer (vm, new_buffers[i]);
d->current_data = s->current_data;
d->current_length = s->current_length;
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
d->flags = s->flags & flag_mask;
}
}
d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
- head_end_offset);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), head_end_offset);
d->next_buffer = src_buffer;
}
vlib_buffer_advance (s, head_end_offset);
tail->total_length_not_including_first_buffer;
next_segment:
- __sync_add_and_fetch (&tail->n_add_refs, 1);
+ clib_atomic_add_fetch (&tail->n_add_refs, 1);
if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
{
/* The provided next_bi buffer index is appended to the end of the packet. */
always_inline vlib_buffer_t *
-vlib_buffer_chain_buffer (vlib_main_t * vm,
- vlib_buffer_t * first,
- vlib_buffer_t * last, u32 next_bi)
+vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi)
{
vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
last->next_buffer = next_bi;
u16 len = clib_min (data_len,
n_buffer_bytes - last->current_length -
last->current_data);
- clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
- len);
+ clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
+ data, len);
vlib_buffer_chain_increase_length (first, last, len);
return len;
}
vlib_buffer_free_list_index_t free_list_index;
u32 *free_buffers;
+
+ u8 *name;
} vlib_packet_template_t;
void vlib_packet_template_get_packet_helper (vlib_main_t * vm,
vec_free (t->packet_data);
}
-always_inline u32
-unserialize_vlib_buffer_n_bytes (serialize_main_t * m)
-{
- serialize_stream_t *s = &m->stream;
- vlib_serialize_buffer_main_t *sm
- = uword_to_pointer (m->stream.data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vlib_main_t *vm = sm->vlib_main;
- u32 n, *f;
-
- n = s->n_buffer_bytes - s->current_buffer_index;
- if (sm->last_buffer != ~0)
- {
- vlib_buffer_t *b = vlib_get_buffer (vm, sm->last_buffer);
- while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
- {
- b = vlib_get_buffer (vm, b->next_buffer);
- n += b->current_length;
- }
- }
-
- /* *INDENT-OFF* */
- clib_fifo_foreach (f, sm->rx.buffer_fifo, ({
- n += vlib_buffer_index_length_in_chain (vm, f[0]);
- }));
-/* *INDENT-ON* */
-
- return n;
-}
-
/* Set a buffer quickly into "uninitialized" state. We want this to
be extremely cheap and arrange for all fields that need to be
initialized to be in the first 128 bits of the buffer. */
/* Make sure buffer template is sane. */
ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
- clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+ clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
/* Not in the first 16 octets. */
dst->n_add_refs = src->n_add_refs;
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
p = hash_get (vlib_buffer_state_validation_hash, b);
oldheap = clib_mem_set_heap (vlib_buffer_state_heap);
- while (__sync_lock_test_and_set (vlib_buffer_state_validation_lock, 1))
+ while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock))
;
hash_set (vlib_buffer_state_validation_hash, b, expected);
vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer);
u32 need = want_first_size - first->current_length;
u32 amount_to_copy = clib_min (need, second->current_length);
- clib_memcpy (((u8 *) vlib_buffer_get_current (first)) +
- first->current_length,
- vlib_buffer_get_current (second), amount_to_copy);
+ clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) +
+ first->current_length,
+ vlib_buffer_get_current (second), amount_to_copy);
first->current_length += amount_to_copy;
- vlib_buffer_advance (second, amount_to_copy);
+ second->current_data += amount_to_copy;
+ second->current_length -= amount_to_copy;
if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
{
first->total_length_not_including_first_buffer -= amount_to_copy;
(first->flags & VLIB_BUFFER_NEXT_PRESENT));
}
+/**
+ * @brief linearize buffer chain - the first buffer is filled, if needed,
+ * buffers are allocated and filled, returns free space in last buffer or
+ * negative on failure
+ *
+ * @param[in] vm - vlib_main
+ * @param[in,out] first - first buffer in chain
+ */
+always_inline int
+vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * first)
+{
+ vlib_buffer_t *b = first;
+ vlib_buffer_free_list_t *fl =
+ vlib_buffer_get_free_list (vm, vlib_buffer_get_free_list_index (b));
+ u32 buf_len = fl->n_data_bytes;
+ // free buffer chain starting from the second buffer
+ int free_count = (b->flags & VLIB_BUFFER_NEXT_PRESENT) != 0;
+ u32 chain_to_free = b->next_buffer;
+
+ u32 len = vlib_buffer_length_in_chain (vm, b);
+ u32 free_len = buf_len - b->current_data - b->current_length;
+ int alloc_len = clib_max (len - free_len, 0); //use the free len in the first buffer
+ int n_buffers = (alloc_len + buf_len - 1) / buf_len;
+ u32 new_buffers[n_buffers];
+
+ u32 n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
+ if (n_alloc != n_buffers)
+ {
+ vlib_buffer_free_no_next (vm, new_buffers, n_alloc);
+ return -1;
+ }
+
+ vlib_buffer_t *s = b;
+ while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ s = vlib_get_buffer (vm, s->next_buffer);
+ int d_free_len = buf_len - b->current_data - b->current_length;
+ ASSERT (d_free_len >= 0);
+ // chain buf and split write
+ u32 copy_len = clib_min (d_free_len, s->current_length);
+ u8 *d = vlib_buffer_put_uninit (b, copy_len);
+ clib_memcpy (d, vlib_buffer_get_current (s), copy_len);
+ int rest = s->current_length - copy_len;
+ if (rest > 0)
+ {
+ //prev buf is full
+ ASSERT (vlib_buffer_get_tail (b) == b->data + buf_len);
+ ASSERT (n_buffers > 0);
+ b = vlib_buffer_chain_buffer (vm, b, new_buffers[--n_buffers]);
+ //make full use of the new buffers
+ b->current_data = 0;
+ d = vlib_buffer_put_uninit (b, rest);
+ clib_memcpy (d, vlib_buffer_get_current (s) + copy_len, rest);
+ }
+ }
+ vlib_buffer_free (vm, &chain_to_free, free_count);
+ b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
+ if (b == first) /* no buffers addeed */
+ b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
+ ASSERT (len == vlib_buffer_length_in_chain (vm, first));
+ ASSERT (n_buffers == 0);
+ return buf_len - b->current_data - b->current_length;
+}
+
#endif /* included_vlib_buffer_funcs_h */
/*
Code Review / vpp.git / blobdiff