- while (n_left >= 4)
- {
- vlib_buffer_t * b0, * b1;
- u16 desc_chain_head0,desc_chain_head1;
- u16 desc_current0,desc_current1;
- uword offset0, offset1;
- u16 bytes_left0, bytes_left1;
- void *buffer_addr0, *buffer_addr1;
-
- vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
- vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
-
- b0 = vlib_get_buffer (vm, buffers[0]);
- b1 = vlib_get_buffer (vm, buffers[1]);
- buffers+=2;
- n_left-=2;
-
- desc_current0 = desc_chain_head0 = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- desc_current1 = desc_chain_head1 = rxvq->avail->ring[(rxvq->last_avail_idx+1) & qsz_mask];
-
- offset0 = vui->virtio_net_hdr_sz;
-
- offset1 = vui->virtio_net_hdr_sz;
-
- bytes_left0 = b0->current_length;
- bytes_left1 = b1->current_length;
-
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
-
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
-
- virtio_net_hdr_mrg_rxbuf_t * hdr0 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr0;
- virtio_net_hdr_mrg_rxbuf_t * hdr1 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr1;
- hdr0->hdr.flags = 0;
- hdr1->hdr.flags = 0;
- hdr0->hdr.gso_type = 0;
- hdr1->hdr.gso_type = 0;
-
- if (vui->virtio_net_hdr_sz == 12) {
- hdr0->num_buffers = 1;
- hdr1->num_buffers = 1;
- }
-
- buffer_addr0 += offset0;
- buffer_addr1 += offset1;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current0].len = vui->virtio_net_hdr_sz;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current1].len = vui->virtio_net_hdr_sz;
-
- while(1) {
- if (rxvq->desc[desc_current0].len - offset0 > 0 ) {
- u16 bytes_to_copy = bytes_left0 > (rxvq->desc[desc_current0].len - offset0) ? (rxvq->desc[desc_current0].len - offset0) : bytes_left0;
- rte_memcpy(buffer_addr0, vlib_buffer_get_current (b0) + b0->current_length - bytes_left0, bytes_to_copy);
- bytes_left0 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT ) {
- offset0 = 0;
- desc_current0 = rxvq->desc[desc_current1].next;
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }
-
- while(1) {
- if (rxvq->desc[desc_current1].len - offset1 > 0 ) {
- u16 bytes_to_copy = bytes_left1 > (rxvq->desc[desc_current1].len - offset1) ? (rxvq->desc[desc_current1].len - offset1) : bytes_left1;
- rte_memcpy(buffer_addr1, vlib_buffer_get_current (b1) + b1->current_length - bytes_left1, bytes_to_copy);
- bytes_left1 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT ) {
- offset1 = 0;
- desc_current1 = rxvq->desc[desc_current1].next;
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }
-
- rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head0;
- rxvq->used->ring[used_index & qsz_mask].len = b0->current_length + vui->virtio_net_hdr_sz;
- used_index+=1;
- rxvq->used->ring[used_index & qsz_mask].id = desc_chain_head1;
- rxvq->used->ring[used_index & qsz_mask].len = b1->current_length + vui->virtio_net_hdr_sz;
- used_index+=1;
- rxvq->last_avail_idx+=2;
- }
-