+ /* asume that somebody will want to add ethernet header on the packet
+ so start with IP header at offset 14 */
+ start_offset = (mode == MEMIF_INTERFACE_MODE_IP) ? 14 : 0;
+
+ /* for S2M rings, we are consumers of packet buffers, and for M2S rings we
+ are producers of empty buffers */
+ cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq->last_tail;
+ last_slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
+ if (cur_slot == last_slot)
+ goto refill;
+ n_slots = last_slot - cur_slot;
+
+ /* construct copy and packet vector out of ring slots */
+ while (n_slots && n_rx_packets < MEMIF_RX_VECTOR_SZ)
+ {
+ u32 dst_off, src_off, n_bytes_left;
+ u16 s0;
+ memif_desc_t *d0;
+ void *mb0;
+ po = ptd->packet_ops + n_rx_packets;
+ n_rx_packets++;
+ po->first_buffer_vec_index = n_buffers++;
+ po->packet_len = 0;
+ src_off = 0;
+ dst_off = start_offset;
+
+ next_slot:
+ CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask],
+ CLIB_CACHE_LINE_BYTES, LOAD);
+ s0 = cur_slot & mask;
+ d0 = &ring->desc[s0];
+ n_bytes_left = d0->length;
+
+ /* slave resets buffer length,
+ * so it can produce full size buffer for master
+ */
+ if (type == MEMIF_RING_M2S)
+ d0->length = mif->run.buffer_size;
+
+ po->packet_len += n_bytes_left;
+ if (PREDICT_FALSE (last_region != d0->region))
+ {
+ last_region_shm = mif->regions[d0->region].shm;
+ last_region = d0->region;
+ }
+ mb0 = last_region_shm + d0->offset;
+
+ do
+ {
+ u32 dst_free = buffer_size - dst_off;
+ if (dst_free == 0)
+ {
+ dst_off = 0;
+ dst_free = buffer_size;
+ n_buffers++;
+ }
+ u32 bytes_to_copy = clib_min (dst_free, n_bytes_left);
+ memif_add_copy_op (ptd, mb0 + src_off, bytes_to_copy, dst_off,
+ n_buffers - 1);
+ n_bytes_left -= bytes_to_copy;
+ src_off += bytes_to_copy;
+ dst_off += bytes_to_copy;
+ }
+ while (PREDICT_FALSE (n_bytes_left));
+
+ cur_slot++;
+ n_slots--;
+ if ((d0->flags & MEMIF_DESC_FLAG_NEXT) && n_slots)
+ {
+ src_off = 0;
+ goto next_slot;
+ }
+ }
+
+ /* allocate free buffers */
+ vec_validate_aligned (ptd->buffers, n_buffers - 1, CLIB_CACHE_LINE_BYTES);
+ n_alloc = vlib_buffer_alloc (vm, ptd->buffers, n_buffers);
+ if (PREDICT_FALSE (n_alloc != n_buffers))
+ {
+ if (n_alloc)
+ vlib_buffer_free (vm, ptd->buffers, n_alloc);
+ vlib_error_count (vm, node->node_index,
+ MEMIF_INPUT_ERROR_BUFFER_ALLOC_FAIL, 1);
+ goto refill;
+ }
+
+ /* copy data */
+ n_left = vec_len (ptd->copy_ops);
+ co = ptd->copy_ops;
+ while (n_left >= 8)
+ {
+ CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
+
+ b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
+ b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
+ b2 = vlib_get_buffer (vm, ptd->buffers[co[2].buffer_vec_index]);
+ b3 = vlib_get_buffer (vm, ptd->buffers[co[3].buffer_vec_index]);
+
+ clib_memcpy_fast (b0->data + co[0].buffer_offset, co[0].data,
+ co[0].data_len);
+ clib_memcpy_fast (b1->data + co[1].buffer_offset, co[1].data,
+ co[1].data_len);
+ clib_memcpy_fast (b2->data + co[2].buffer_offset, co[2].data,
+ co[2].data_len);
+ clib_memcpy_fast (b3->data + co[3].buffer_offset, co[3].data,
+ co[3].data_len);
+
+ co += 4;
+ n_left -= 4;
+ }
+ while (n_left)
+ {
+ b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
+ clib_memcpy_fast (b0->data + co[0].buffer_offset, co[0].data,
+ co[0].data_len);
+ co += 1;
+ n_left -= 1;
+ }
+
+ /* release slots from the ring */
+ if (type == MEMIF_RING_S2M)