- clib_memcpy (mb0 + total, vlib_buffer_get_current (b0),
- CLIB_CACHE_LINE_BYTES);
- if (len > CLIB_CACHE_LINE_BYTES)
- clib_memcpy (mb0 + CLIB_CACHE_LINE_BYTES + total,
- vlib_buffer_get_current (b0) + CLIB_CACHE_LINE_BYTES,
- len - CLIB_CACHE_LINE_BYTES);
- total += len;
+ if (PREDICT_FALSE (dst_left == 0))
+ {
+ if (free_slots)
+ {
+ slot++;
+ free_slots--;
+ d0->flags = MEMIF_DESC_FLAG_NEXT;
+ d0 = &ring->desc[slot & mask];
+ dst_off = 0;
+ dst_left =
+ (type ==
+ MEMIF_RING_S2M) ? mif->run.buffer_size : d0->length;
+
+ if (PREDICT_FALSE (last_region != d0->region))
+ {
+ last_region_shm = mif->regions[d0->region].shm;
+ last_region = d0->region;
+ }
+ mb0 = last_region_shm + d0->offset;
+ }
+ else
+ {
+ /* we need to rollback vectors before bailing out */
+ _vec_len (ptd->buffers) = saved_ptd_buffers_len;
+ _vec_len (ptd->copy_ops) = saved_ptd_copy_ops_len;
+ vlib_error_count (vm, node->node_index,
+ MEMIF_TX_ERROR_ROLLBACK, 1);
+ slot = saved_slot;
+ goto no_free_slots;
+ }
+ }
+ bytes_to_copy = clib_min (src_left, dst_left);
+ memif_add_copy_op (ptd, mb0 + dst_off, bytes_to_copy, src_off,
+ vec_len (ptd->buffers));
+ vec_add1_aligned (ptd->buffers, bi0, CLIB_CACHE_LINE_BYTES);
+ src_off += bytes_to_copy;
+ dst_off += bytes_to_copy;
+ src_left -= bytes_to_copy;
+ dst_left -= bytes_to_copy;