- if (PREDICT_FALSE((rxvq->avail->idx == rxvq->last_avail_idx) ||
- vui->sock_errno != 0)) {
- vlib_simple_counter_main_t * cm;
- vnet_main_t * vnm = vnet_get_main();
-
- cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, os_get_cpu_number(),
- 0, frame->n_vectors);
-
- vlib_error_count (vm, node->node_index,
- VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF,
- frame->n_vectors);
- goto done2;
- }
-
- if (PREDICT_TRUE(rxvq->avail->idx > rxvq->last_avail_idx))
- n_avail_desc = rxvq->avail->idx - rxvq->last_avail_idx;
- else /* wrapped */
- n_avail_desc = (u16) -1 - rxvq->last_avail_idx + rxvq->avail->idx;
-
- DBG_VQ("rxvq->avail->idx %d rxvq->last_avail_idx %d n_avail_desc %d",
- rxvq->avail->idx, rxvq->last_avail_idx, n_avail_desc);
-
- n_left = n_packets = frame->n_vectors;
- if (PREDICT_FALSE(n_packets > n_avail_desc)) {
- vlib_simple_counter_main_t * cm;
- vnet_main_t * vnm = vnet_get_main();
-
- cm = vec_elt_at_index (vnm->interface_main.sw_if_counters,
- VNET_INTERFACE_COUNTER_TX_ERROR);
- vlib_increment_simple_counter (cm, os_get_cpu_number(),
- 0, frame->n_vectors);
-
- vlib_error_count (vm, node->node_index,
- VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF,
- n_packets - n_avail_desc);
- n_left = n_packets = n_avail_desc;
- }
-
- used_index = rxvq->used->idx;
- qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */
-
- while (n_left >= 4)
- {
- vlib_buffer_t * b0, * b1;
- u16 desc_chain_head0,desc_chain_head1;
- u16 desc_current0,desc_current1;
- uword offset0, offset1;
- u16 bytes_left0, bytes_left1;
- void *buffer_addr0, *buffer_addr1;
-
- vlib_prefetch_buffer_with_index (vm, buffers[2], LOAD);
- vlib_prefetch_buffer_with_index (vm, buffers[3], LOAD);
-
- b0 = vlib_get_buffer (vm, buffers[0]);
- b1 = vlib_get_buffer (vm, buffers[1]);
- buffers+=2;
- n_left-=2;
-
- desc_current0 = desc_chain_head0 = rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
- desc_current1 = desc_chain_head1 = rxvq->avail->ring[(rxvq->last_avail_idx+1) & qsz_mask];
-
- offset0 = vui->virtio_net_hdr_sz;
-
- offset1 = vui->virtio_net_hdr_sz;
-
- bytes_left0 = b0->current_length;
- bytes_left1 = b1->current_length;
-
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
-
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
-
- virtio_net_hdr_mrg_rxbuf_t * hdr0 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr0;
- virtio_net_hdr_mrg_rxbuf_t * hdr1 = (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr1;
- hdr0->hdr.flags = 0;
- hdr1->hdr.flags = 0;
- hdr0->hdr.gso_type = 0;
- hdr1->hdr.gso_type = 0;
-
- if (vui->virtio_net_hdr_sz == 12) {
- hdr0->num_buffers = 1;
- hdr1->num_buffers = 1;
- }
-
- buffer_addr0 += offset0;
- buffer_addr1 += offset1;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current0].len = vui->virtio_net_hdr_sz;
-
- if (PREDICT_FALSE(!vui->is_any_layout && rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT))
- rxvq->desc[desc_current1].len = vui->virtio_net_hdr_sz;
-
- while(1) {
- if (rxvq->desc[desc_current0].len - offset0 > 0 ) {
- u16 bytes_to_copy = bytes_left0 > (rxvq->desc[desc_current0].len - offset0) ? (rxvq->desc[desc_current0].len - offset0) : bytes_left0;
- rte_memcpy(buffer_addr0, vlib_buffer_get_current (b0) + b0->current_length - bytes_left0, bytes_to_copy);
- bytes_left0 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current0].flags & VIRTQ_DESC_F_NEXT ) {
- offset0 = 0;
- desc_current0 = rxvq->desc[desc_current1].next;
- buffer_addr0 = map_guest_mem(vui, rxvq->desc[desc_current0].addr);
- if (PREDICT_FALSE(!buffer_addr0)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }
-
- while(1) {
- if (rxvq->desc[desc_current1].len - offset1 > 0 ) {
- u16 bytes_to_copy = bytes_left1 > (rxvq->desc[desc_current1].len - offset1) ? (rxvq->desc[desc_current1].len - offset1) : bytes_left1;
- rte_memcpy(buffer_addr1, vlib_buffer_get_current (b1) + b1->current_length - bytes_left1, bytes_to_copy);
- bytes_left1 -= bytes_to_copy;
- }
-
- if (rxvq->desc[desc_current1].flags & VIRTQ_DESC_F_NEXT ) {
- offset1 = 0;
- desc_current1 = rxvq->desc[desc_current1].next;
- buffer_addr1 = map_guest_mem(vui, rxvq->desc[desc_current1].addr);
- if (PREDICT_FALSE(!buffer_addr1)) {
- vlib_error_count (vm, node->node_index, VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL, 1);
- goto done;
- }
- }
- else
- break;
- }