X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fvmxnet3%2Finput.c;h=f182409ac593a106f8ab4af2e511d9c733ed5099;hb=007abe751f2ee86528d0ccc005a3da1c90850868;hp=a5a5d2f43b857922e46c2b2b51d9f3ed9e734ed3;hpb=df7f8e8cffcc43531f7daeda44d436b60e538141;p=vpp.git diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c index a5a5d2f43b8..f182409ac59 100644 --- a/src/plugins/vmxnet3/input.c +++ b/src/plugins/vmxnet3/input.c @@ -20,12 +20,17 @@ #include #include #include +#include +#include +#include #include #define foreach_vmxnet3_input_error \ _(BUFFER_ALLOC, "buffer alloc error") \ + _(RX_PACKET_NO_SOP, "Rx packet error - no SOP") \ _(RX_PACKET, "Rx packet error") \ + _(RX_PACKET_EOP, "Rx packet error found on EOP") \ _(NO_BUFFER, "Rx no buffer error") typedef enum @@ -69,6 +74,128 @@ vmxnet3_rx_comp_ring_advance_next (vmxnet3_rxq_t * rxq) } } +static_always_inline void +vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, + u16 gso_size) +{ + u8 l4_hdr_sz = 0; + + if (rx_comp->flags & VMXNET3_RXCF_IP4) + { + ip4_header_t *ip4 = (ip4_header_t *) (hb->data + + sizeof (ethernet_header_t)); + + vnet_buffer (hb)->l2_hdr_offset = 0; + vnet_buffer (hb)->l3_hdr_offset = sizeof (ethernet_header_t); + vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + + ip4_header_bytes (ip4); + hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4; + + /* checksum offload */ + if (!(rx_comp->index & VMXNET3_RXCI_CNC)) + { + if (!(rx_comp->flags & VMXNET3_RXCF_IPC)) + { + hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + ip4->checksum = 0; + } + if (!(rx_comp->flags & VMXNET3_RXCF_TUC)) + { + if (rx_comp->flags & VMXNET3_RXCF_TCP) + { + tcp_header_t *tcp = + (tcp_header_t *) (hb->data + + vnet_buffer (hb)->l4_hdr_offset); + hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + tcp->checksum = 0; + } + else if (rx_comp->flags & VMXNET3_RXCF_UDP) + { + udp_header_t *udp = + (udp_header_t *) (hb->data + + vnet_buffer (hb)->l4_hdr_offset); + hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + udp->checksum = 0; + } + } + } + + if (gso_size) + { + if (rx_comp->flags & VMXNET3_RXCF_TCP) + { + tcp_header_t *tcp = + (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); + l4_hdr_sz = tcp_header_bytes (tcp); + } + else if (rx_comp->flags & VMXNET3_RXCF_UDP) + { + udp_header_t *udp = + (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); + l4_hdr_sz = sizeof (*udp); + } + vnet_buffer2 (hb)->gso_size = gso_size; + vnet_buffer2 (hb)->gso_l4_hdr_sz = l4_hdr_sz; + hb->flags |= VNET_BUFFER_F_GSO; + } + } + else if (rx_comp->flags & VMXNET3_RXCF_IP6) + { + vnet_buffer (hb)->l2_hdr_offset = 0; + vnet_buffer (hb)->l3_hdr_offset = sizeof (ethernet_header_t); + vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + + sizeof (ip6_header_t); + hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6; + + /* checksum offload */ + if (!(rx_comp->index & VMXNET3_RXCI_CNC)) + { + if (!(rx_comp->flags & VMXNET3_RXCF_TUC)) + { + if (rx_comp->flags & VMXNET3_RXCF_TCP) + { + tcp_header_t *tcp = + (tcp_header_t *) (hb->data + + vnet_buffer (hb)->l4_hdr_offset); + hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + tcp->checksum = 0; + } + else if (rx_comp->flags & VMXNET3_RXCF_UDP) + { + udp_header_t *udp = + (udp_header_t *) (hb->data + + vnet_buffer (hb)->l4_hdr_offset); + hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + udp->checksum = 0; + } + } + } + + if (gso_size) + { + if (rx_comp->flags & VMXNET3_RXCF_TCP) + { + tcp_header_t *tcp = + (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); + l4_hdr_sz = tcp_header_bytes (tcp); + } + else if (rx_comp->flags & VMXNET3_RXCF_UDP) + { + udp_header_t *udp = + (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); + l4_hdr_sz = sizeof (*udp); + } + vnet_buffer2 (hb)->gso_size = gso_size; + vnet_buffer2 (hb)->gso_l4_hdr_sz = l4_hdr_sz; + hb->flags |= VNET_BUFFER_F_GSO; + } + } +} + static_always_inline uword vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame, vmxnet3_device_t * vd, @@ -78,7 +205,6 @@ vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, uword n_trace = vlib_get_trace_count (vm, node); u32 n_rx_packets = 0, n_rx_bytes = 0; vmxnet3_rx_comp *rx_comp; - u32 comp_idx; u32 desc_idx; vmxnet3_rxq_t *rxq; u32 thread_index = vm->thread_index; @@ -88,18 +214,24 @@ vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vmxnet3_rx_comp_ring *comp_ring; u16 rid; vlib_buffer_t *prev_b0 = 0, *hb = 0; + u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + u8 known_next = 0, got_packet = 0; + vmxnet3_rx_desc *rxd; + clib_error_t *error; + u16 gso_size = 0; rxq = vec_elt_at_index (vd->rxqs, qid); comp_ring = &rxq->rx_comp_ring; bi = buffer_indices; next = nexts; - while (comp_ring->gen == - (rxq->rx_comp[comp_ring->next].flags & VMXNET3_RXCF_GEN)) + rx_comp = &rxq->rx_comp[comp_ring->next]; + + while (PREDICT_TRUE ((n_rx_packets < VLIB_FRAME_SIZE) && + (comp_ring->gen == + (rx_comp->flags & VMXNET3_RXCF_GEN)))) { vlib_buffer_t *b0; - - comp_idx = comp_ring->next; - rx_comp = &rxq->rx_comp[comp_idx]; + u32 bi0; rid = vmxnet3_find_rid (vd, rx_comp); ring = &rxq->rx_ring[rid]; @@ -110,35 +242,66 @@ vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { vlib_error_count (vm, node->node_index, VMXNET3_INPUT_ERROR_NO_BUFFER, 1); + if (hb) + { + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb)); + hb = 0; + } + prev_b0 = 0; break; } - vmxnet3_rx_comp_ring_advance_next (rxq); desc_idx = rx_comp->index & VMXNET3_RXC_INDEX; ring->consume = desc_idx; + rxd = &rxq->rx_desc[rid][desc_idx]; - bi[0] = ring->bufs[desc_idx]; + bi0 = ring->bufs[desc_idx]; ring->bufs[desc_idx] = ~0; - b0 = vlib_get_buffer (vm, bi[0]); + b0 = vlib_get_buffer (vm, bi0); vnet_buffer (b0)->sw_if_index[VLIB_RX] = vd->sw_if_index; vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; + vnet_buffer (b0)->feature_arc_index = 0; b0->current_length = rx_comp->len & VMXNET3_RXCL_LEN_MASK; b0->current_data = 0; b0->total_length_not_including_first_buffer = 0; b0->next_buffer = 0; b0->flags = 0; b0->error = 0; - ASSERT (b0->current_length != 0); + b0->current_config_index = 0; + + if (PREDICT_FALSE ((rx_comp->index & VMXNET3_RXCI_EOP) && + (rx_comp->len & VMXNET3_RXCL_ERROR))) + { + vlib_buffer_free_one (vm, bi0); + vlib_error_count (vm, node->node_index, + VMXNET3_INPUT_ERROR_RX_PACKET_EOP, 1); + if (hb && vlib_get_buffer_index (vm, hb) != bi0) + { + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb)); + hb = 0; + } + prev_b0 = 0; + goto next; + } if (rx_comp->index & VMXNET3_RXCI_SOP) { + ASSERT (!(rxd->flags & VMXNET3_RXF_BTYPE)); /* start segment */ + if (vd->gso_enable && + (rx_comp->flags & VMXNET3_RXCF_CT) == VMXNET3_RXCOMP_TYPE_LRO) + { + vmxnet3_rx_comp_ext *lro = (vmxnet3_rx_comp_ext *) rx_comp; + + gso_size = lro->flags & VMXNET3_RXECF_MSS_MASK; + } + hb = b0; + bi[0] = bi0; if (!(rx_comp->index & VMXNET3_RXCI_EOP)) { hb->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; - b0->flags |= VLIB_BUFFER_NEXT_PRESENT; prev_b0 = b0; } else @@ -147,54 +310,104 @@ vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, * Both start and end of packet is set. It is a complete packet */ prev_b0 = 0; + got_packet = 1; } - } else if (rx_comp->index & VMXNET3_RXCI_EOP) { /* end of segment */ - if (prev_b0) + if (PREDICT_TRUE (prev_b0 != 0)) { - prev_b0->next_buffer = bi[0]; - prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT; - hb->total_length_not_including_first_buffer += - b0->current_length; - prev_b0 = 0; // Get next packet + if (PREDICT_TRUE (b0->current_length != 0)) + { + prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT; + prev_b0->next_buffer = bi0; + hb->total_length_not_including_first_buffer += + b0->current_length; + } + else + { + vlib_buffer_free_one (vm, bi0); + } + prev_b0 = 0; + got_packet = 1; } else { /* EOP without SOP, error */ - hb = 0; - bi++; vlib_error_count (vm, node->node_index, - VMXNET3_INPUT_ERROR_RX_PACKET, 1); - vlib_buffer_free_one (vm, bi[0]); - continue; + VMXNET3_INPUT_ERROR_RX_PACKET_NO_SOP, 1); + vlib_buffer_free_one (vm, bi0); + if (hb && vlib_get_buffer_index (vm, hb) != bi0) + { + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb)); + hb = 0; + } + goto next; } } else if (prev_b0) // !sop && !eop { /* mid chain */ - b0->flags |= VLIB_BUFFER_NEXT_PRESENT; - prev_b0->next_buffer = bi[0]; + ASSERT (rxd->flags & VMXNET3_RXF_BTYPE); + prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT; + prev_b0->next_buffer = bi0; prev_b0 = b0; hb->total_length_not_including_first_buffer += b0->current_length; } else { - ASSERT (0); + vlib_error_count (vm, node->node_index, + VMXNET3_INPUT_ERROR_RX_PACKET, 1); + vlib_buffer_free_one (vm, bi0); + if (hb && vlib_get_buffer_index (vm, hb) != bi0) + { + vlib_buffer_free_one (vm, vlib_get_buffer_index (vm, hb)); + hb = 0; + } + goto next; } - bi++; n_rx_bytes += b0->current_length; - if (!prev_b0) + if (got_packet) { - next[0] = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + if (PREDICT_FALSE (vd->per_interface_next_index != ~0)) + { + next_index = vd->per_interface_next_index; + known_next = 1; + } + + if (PREDICT_FALSE + (vnet_device_input_have_features (vd->sw_if_index))) + { + vnet_feature_start_device_input_x1 (vd->sw_if_index, + &next_index, hb); + known_next = 1; + } + + if (PREDICT_FALSE (known_next)) + next[0] = next_index; + else + { + ethernet_header_t *e = (ethernet_header_t *) hb->data; + + next[0] = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + if (!ethernet_frame_is_tagged (ntohs (e->type))) + vmxnet3_handle_offload (rx_comp, hb, gso_size); + } + n_rx_packets++; next++; + bi++; hb = 0; + got_packet = 0; + gso_size = 0; } + + next: + vmxnet3_rx_comp_ring_advance_next (rxq); + rx_comp = &rxq->rx_comp[comp_ring->next]; } if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)))) @@ -225,27 +438,25 @@ vmxnet3_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_TRUE (n_rx_packets)) { - clib_error_t *error; - vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_rx_packets); vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, - vd->hw_if_index, n_rx_packets, n_rx_bytes); + vd->sw_if_index, n_rx_packets, n_rx_bytes); + } - error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq); - if (PREDICT_FALSE (error != 0)) - { - vlib_error_count (vm, node->node_index, - VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1); - } - error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq); - if (PREDICT_FALSE (error != 0)) - { - vlib_error_count (vm, node->node_index, - VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1); - } + error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq); + if (PREDICT_FALSE (error != 0)) + { + vlib_error_count (vm, node->node_index, + VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1); + } + error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq); + if (PREDICT_FALSE (error != 0)) + { + vlib_error_count (vm, node->node_index, + VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1); } return n_rx_packets; @@ -276,6 +487,7 @@ VLIB_NODE_FN (vmxnet3_input_node) (vlib_main_t * vm, VLIB_REGISTER_NODE (vmxnet3_input_node) = { .name = "vmxnet3-input", .sibling_of = "device-input", + .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, .format_trace = format_vmxnet3_input_trace, .type = VLIB_NODE_TYPE_INPUT, .state = VLIB_NODE_STATE_DISABLED,