From 9e2a78564f0fc07f1ea3d15a31fa7ca3a0f6424d Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Thu, 13 Aug 2020 18:57:26 +0200 Subject: [PATCH] tap: add gro support Type: feature Change-Id: I5868dd267aa26aa97aec5fd70e70c5956ac52277 Signed-off-by: Mohsin Kazmi --- src/vnet/devices/tap/cli.c | 15 ++++++++++----- src/vnet/devices/tap/tap.c | 16 +++++++++++++++- src/vnet/devices/tap/tap.h | 2 +- src/vnet/devices/virtio/device.c | 33 ++++++++++++++++++++++++++------- src/vnet/devices/virtio/node.c | 9 +++++++++ src/vnet/devices/virtio/pci.c | 1 + src/vnet/devices/virtio/virtio.c | 21 +++++++++++++++++++++ src/vnet/devices/virtio/virtio.h | 4 ++++ 8 files changed, 87 insertions(+), 14 deletions(-) diff --git a/src/vnet/devices/tap/cli.c b/src/vnet/devices/tap/cli.c index 34897ea7c5d..fa5fa91e7a6 100644 --- a/src/vnet/devices/tap/cli.c +++ b/src/vnet/devices/tap/cli.c @@ -201,7 +201,7 @@ tap_offload_command_fn (vlib_main_t * vm, unformat_input_t * input, unformat_input_t _line_input, *line_input = &_line_input; u32 sw_if_index = ~0; vnet_main_t *vnm = vnet_get_main (); - int gso_enable = 0, gso_disable = 0; + int gso_enable = 0, gso_disable = 0, is_gro_coalesce = 0; int csum_offload_enable = 0, csum_offload_disable = 0; int rv = 0; @@ -217,7 +217,11 @@ tap_offload_command_fn (vlib_main_t * vm, unformat_input_t * input, vnm, &sw_if_index)) ; else if (unformat (line_input, "gso-enable")) - gso_enable = 1; + { + gso_enable = 1; + if (unformat (line_input, "gro-coalesce")) + is_gro_coalesce = 1; + } else if (unformat (line_input, "gso-disable")) gso_disable = 1; else if (unformat (line_input, "csum-offload-enable")) @@ -235,11 +239,11 @@ tap_offload_command_fn (vlib_main_t * vm, unformat_input_t * input, "please specify interface name or sw_if_index"); if (gso_enable) - rv = tap_gso_enable_disable (vm, sw_if_index, 1); + rv = tap_gso_enable_disable (vm, sw_if_index, 1, is_gro_coalesce); else if (csum_offload_enable) rv = tap_csum_offload_enable_disable (vm, sw_if_index, 1); else if (gso_disable) - rv = tap_gso_enable_disable (vm, sw_if_index, 0); + rv = tap_gso_enable_disable (vm, sw_if_index, 0, 0); else if (csum_offload_disable) rv = tap_csum_offload_enable_disable (vm, sw_if_index, 0); @@ -256,7 +260,8 @@ VLIB_CLI_COMMAND (tap_offload_command, static) = { .path = "set tap offload", .short_help = "set tap offload { | sw_if_index }" - " ", + " ", .function = tap_offload_command_fn, }; /* *INDENT-ON* */ diff --git a/src/vnet/devices/tap/tap.c b/src/vnet/devices/tap/tap.c index fa5eef19c0d..a76b4e0c16a 100644 --- a/src/vnet/devices/tap/tap.c +++ b/src/vnet/devices/tap/tap.c @@ -730,6 +730,12 @@ tap_create_if (vlib_main_t * vm, tap_create_if_args_t * args) { hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD; } + if ((args->tap_flags & TAP_FLAG_GSO) + && (args->tap_flags & TAP_FLAG_GRO_COALESCE)) + { + vif->packet_coalesce = 1; + virtio_set_packet_coalesce (vif); + } vnet_hw_interface_set_input_node (vnm, vif->hw_if_index, virtio_input_node.index); @@ -827,6 +833,7 @@ tap_csum_offload_enable_disable (vlib_main_t * vm, u32 sw_if_index, vec_foreach_index (i, vif->tap_fds) _IOCTL (vif->tap_fds[i], TUNSETOFFLOAD, offload); vif->gso_enabled = 0; + vif->packet_coalesce = 0; vif->csum_offload_enabled = enable_disable ? 1 : 0; if ((hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) != 0) @@ -862,7 +869,8 @@ error: } int -tap_gso_enable_disable (vlib_main_t * vm, u32 sw_if_index, int enable_disable) +tap_gso_enable_disable (vlib_main_t * vm, u32 sw_if_index, int enable_disable, + int is_packet_coalesce) { vnet_main_t *vnm = vnet_get_main (); virtio_main_t *mm = &virtio_main; @@ -892,6 +900,11 @@ tap_gso_enable_disable (vlib_main_t * vm, u32 sw_if_index, int enable_disable) hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO | VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD; } + if (is_packet_coalesce) + { + vif->packet_coalesce = 1; + virtio_set_packet_coalesce (vif); + } } else { @@ -900,6 +913,7 @@ tap_gso_enable_disable (vlib_main_t * vm, u32 sw_if_index, int enable_disable) hw->flags &= ~(VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO | VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD); } + vif->packet_coalesce = 0; } error: diff --git a/src/vnet/devices/tap/tap.h b/src/vnet/devices/tap/tap.h index a9cd4766c3c..93b5da51dc5 100644 --- a/src/vnet/devices/tap/tap.h +++ b/src/vnet/devices/tap/tap.h @@ -101,7 +101,7 @@ typedef struct void tap_create_if (vlib_main_t * vm, tap_create_if_args_t * args); int tap_delete_if (vlib_main_t * vm, u32 sw_if_index); int tap_gso_enable_disable (vlib_main_t * vm, u32 sw_if_index, - int enable_disable); + int enable_disable, int packet_coalesce); int tap_csum_offload_enable_disable (vlib_main_t * vm, u32 sw_if_index, int enable_disable); diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 45c390d415d..99157cd6166 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -502,7 +503,7 @@ static_always_inline uword virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame, virtio_if_t * vif, virtio_if_type_t type, int do_gso, - int csum_offload) + int csum_offload, int do_gro) { u16 n_left = frame->n_vectors; virtio_vring_t *vring; @@ -513,6 +514,7 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u16 mask = sz - 1; u16 retry_count = 2; u32 *buffers = vlib_frame_vector_args (frame); + u32 to[GRO_TO_VECTOR_SIZE (n_left)]; clib_spinlock_lock_if_init (&vring->lockp); @@ -520,6 +522,12 @@ virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, (vring->last_kick_avail_idx != vring->avail->idx)) virtio_kick (vm, vring, vif); + if (do_gro) + { + n_left = vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to); + buffers = to; + } + retry: /* free consumed buffers */ virtio_free_used_device_desc (vm, vring, node->node_index); @@ -621,7 +629,8 @@ retry: if (retry_count--) goto retry; - virtio_interface_drop_inline (vm, node->node_index, buffers, n_left, + virtio_interface_drop_inline (vm, node->node_index, + buffers, n_left, VIRTIO_TX_ERROR_NO_FREE_SLOTS); } @@ -641,15 +650,18 @@ virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 1 /* do_gso */ , - 1 /* checksum offload */ ); + 1 /* checksum offload */ , + vif->packet_coalesce); else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD) return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 0 /* no do_gso */ , - 1 /* checksum offload */ ); + 1 /* checksum offload */ , + 0 /* do_gro */ ); else return virtio_interface_tx_gso_inline (vm, node, frame, vif, type, 0 /* no do_gso */ , - 0 /* no checksum offload */ ); + 0 /* no checksum offload */ , + 0 /* do_gro */ ); } VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm, @@ -717,9 +729,16 @@ virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid, } if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING) - vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + { + /* only enable packet coalesce in poll mode */ + gro_flow_table_set_is_enable (vring->flow_table, 1); + vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; + } else - vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; + { + gro_flow_table_set_is_enable (vring->flow_table, 0); + vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; + } return 0; } diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 63e40da53b7..a27131a6398 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -267,6 +268,8 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 thread_index = vm->thread_index; uword n_trace = vlib_get_trace_count (vm, node); virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid); + u16 txq_id = thread_index % vif->num_txqs; + virtio_vring_t *txq_vring = vec_elt_at_index (vif->txq_vrings, txq_id); u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; const int hdr_sz = vif->virtio_net_hdr_sz; u32 *to_next = 0; @@ -276,6 +279,12 @@ virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u16 last = vring->last_used_idx; u16 n_left = vring->used->idx - last; + if (vif->packet_coalesce) + { + vnet_gro_flow_table_schedule_node_on_dispatcher (vm, + txq_vring->flow_table); + } + if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 && vring->last_kick_avail_idx != vring->avail->idx) virtio_kick (vm, vring, vif); diff --git a/src/vnet/devices/virtio/pci.c b/src/vnet/devices/virtio/pci.c index cb3a3a73632..d28c7464e5a 100644 --- a/src/vnet/devices/virtio/pci.c +++ b/src/vnet/devices/virtio/pci.c @@ -824,6 +824,7 @@ virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) vring->used = vr.used; vring->queue_id = queue_num; vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; + vring->flow_table = 0; ASSERT (vring->buffers == 0); vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c index 17fbd7920da..f8b6a39ea47 100644 --- a/src/vnet/devices/virtio/virtio.c +++ b/src/vnet/devices/virtio/virtio.c @@ -217,10 +217,25 @@ virtio_vring_free_tx (vlib_main_t * vm, virtio_if_t * vif, u32 idx) if (vring->avail) clib_mem_free (vring->avail); vec_free (vring->buffers); + gro_flow_table_free (vring->flow_table); clib_spinlock_free (&vring->lockp); return 0; } +void +virtio_set_packet_coalesce (virtio_if_t * vif) +{ + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); + virtio_vring_t *vring; + vec_foreach (vring, vif->txq_vrings) + { + gro_flow_table_init (&vring->flow_table, + vif->type & (VIRTIO_IF_TYPE_TAP | + VIRTIO_IF_TYPE_PCI), hw->tx_node_index); + } +} + void virtio_vring_set_numa_node (vlib_main_t * vm, virtio_if_t * vif, u32 idx) { @@ -320,6 +335,7 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) } vlib_cli_output (vm, " gso-enabled %d", vif->gso_enabled); vlib_cli_output (vm, " csum-enabled %d", vif->csum_offload_enabled); + vlib_cli_output (vm, " packet-coalesce %d", vif->packet_coalesce); if (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI)) vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address, vif->mac_addr); @@ -412,6 +428,11 @@ virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type) vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd, vring->call_fd); } + if (vring->flow_table) + { + vlib_cli_output (vm, " %U", gro_flow_table_format, + vring->flow_table); + } if (show_descr) { vlib_cli_output (vm, "\n descriptor table:\n"); diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h index 04896ed6e2a..b57c6ebd77d 100644 --- a/src/vnet/devices/virtio/virtio.h +++ b/src/vnet/devices/virtio/virtio.h @@ -22,6 +22,7 @@ #include #include #include +#include #define foreach_virtio_net_features \ _ (VIRTIO_NET_F_CSUM, 0) /* Host handles pkts w/ partial csum */ \ @@ -123,6 +124,7 @@ typedef struct u16 last_used_idx; u16 last_kick_avail_idx; u32 call_file_index; + gro_flow_table_t *flow_table; } virtio_vring_t; typedef union @@ -165,6 +167,7 @@ typedef struct u32 sw_if_index; CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); + int packet_coalesce; union { u32 id; @@ -221,6 +224,7 @@ extern void virtio_free_rx_buffers (vlib_main_t * vm, virtio_vring_t * vring); extern void virtio_set_net_hdr_size (virtio_if_t * vif); extern void virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type); +extern void virtio_set_packet_coalesce (virtio_if_t * vif); extern void virtio_pci_legacy_notify_queue (vlib_main_t * vm, virtio_if_t * vif, u16 queue_id); format_function_t format_virtio_device_name; -- 2.16.6