From 426a306cc10559275b809d68f3045deb80f928cb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Beno=C3=AEt=20Ganne?= Date: Wed, 25 Nov 2020 10:41:26 +0100 Subject: [PATCH] af_xdp: update interrupt mode to new infra MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Type: improvement Change-Id: Icb23af5f5e458a555f416cb0a829e84646b25dd9 Signed-off-by: Benoît Ganne --- src/plugins/af_xdp/af_xdp.h | 14 +++++---- src/plugins/af_xdp/device.c | 75 ++++++++++++++++++++++++++++++++++++--------- src/plugins/af_xdp/input.c | 33 ++++++++++---------- 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/src/plugins/af_xdp/af_xdp.h b/src/plugins/af_xdp/af_xdp.h index 3bd53ad768d..568380baa56 100644 --- a/src/plugins/af_xdp/af_xdp.h +++ b/src/plugins/af_xdp/af_xdp.h @@ -27,12 +27,12 @@ #define af_xdp_log(lvl, dev, f, ...) \ vlib_log(lvl, af_xdp_main.log_class, "%v: " f, (dev)->name, ##__VA_ARGS__) -#define foreach_af_xdp_device_flags \ - _(0, INITIALIZED, "initialized") \ - _(1, ERROR, "error") \ - _(2, ADMIN_UP, "admin-up") \ - _(4, LINK_UP, "link-up") \ - _(8, ZEROCOPY, "zero-copy") \ +#define foreach_af_xdp_device_flags \ + _ (0, INITIALIZED, "initialized") \ + _ (1, ERROR, "error") \ + _ (2, ADMIN_UP, "admin-up") \ + _ (3, LINK_UP, "link-up") \ + _ (4, ZEROCOPY, "zero-copy") enum { @@ -62,6 +62,8 @@ typedef struct /* fields below are accessed in control-plane only (cold) */ uword file_index; + u32 queue_index; + u8 is_polling; } af_xdp_rxq_t; typedef struct diff --git a/src/plugins/af_xdp/device.c b/src/plugins/af_xdp/device.c index 5090d3a649a..632f1608c1e 100644 --- a/src/plugins/af_xdp/device.c +++ b/src/plugins/af_xdp/device.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "af_xdp.h" af_xdp_main_t af_xdp_main; @@ -98,7 +99,6 @@ af_xdp_delete_if (vlib_main_t * vm, af_xdp_device_t * ad) if (ad->hw_if_index) { vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0); - vnet_hw_interface_unassign_rx_thread (vnm, ad->hw_if_index, 0); ethernet_delete_interface (vnm, ad->hw_if_index); } @@ -292,15 +292,25 @@ af_xdp_get_numa (const char *ifname) static clib_error_t * af_xdp_device_rxq_read_ready (clib_file_t * f) { - vnet_main_t *vnm = vnet_get_main (); - const af_xdp_main_t *am = &af_xdp_main; - const u32 dev_instance = f->private_data >> 16; - const u16 qid = f->private_data & 0xffff; - const af_xdp_device_t *ad = vec_elt_at_index (am->devices, dev_instance); - vnet_device_input_set_interrupt_pending (vnm, ad->hw_if_index, qid); + vnet_hw_if_rx_queue_set_int_pending (vnet_get_main (), f->private_data); return 0; } +static void +af_xdp_device_set_rxq_mode (af_xdp_rxq_t *rxq, int is_polling) +{ + clib_file_main_t *fm = &file_main; + clib_file_t *f; + + if (rxq->is_polling == is_polling) + return; + + f = clib_file_get (fm, rxq->file_index); + fm->file_update (f, is_polling ? UNIX_FILE_UPDATE_DELETE : + UNIX_FILE_UPDATE_ADD); + rxq->is_polling = !!is_polling; +} + void af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args) { @@ -418,19 +428,25 @@ af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args) for (i = 0; i < vec_len (ad->rxqs); i++) { af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i); + rxq->queue_index = vnet_hw_if_register_rx_queue ( + vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY); + u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name, + ad->dev_instance, i); clib_file_t f = { .file_descriptor = rxq->xsk_fd, .flags = UNIX_FILE_EVENT_EDGE_TRIGGERED, - .private_data = (uword) ad->dev_instance << 16 | (uword) i, + .private_data = rxq->queue_index, .read_function = af_xdp_device_rxq_read_ready, - .description = - format (0, "%U rxq %d", format_af_xdp_device_name, ad->dev_instance, - i), + .description = desc, }; rxq->file_index = clib_file_add (&file_main, &f); - vnet_hw_interface_assign_rx_thread (vnm, ad->hw_if_index, i, ~0); + vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index, + rxq->file_index); + af_xdp_device_set_rxq_mode (rxq, 1 /* polling */); } + vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index); + /* buffer template */ vec_validate_aligned (ad->buffer_template, 1, CLIB_CACHE_LINE_BYTES); ad->buffer_template->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -472,6 +488,37 @@ af_xdp_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) return 0; } +static clib_error_t * +af_xdp_interface_rx_mode_change (vnet_main_t *vnm, u32 hw_if_index, u32 qid, + vnet_hw_if_rx_mode mode) +{ + af_xdp_main_t *am = &af_xdp_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance); + af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid); + + switch (mode) + { + case VNET_HW_IF_RX_MODE_UNKNOWN: + case VNET_HW_IF_NUM_RX_MODES: /* fallthrough */ + return clib_error_create ("uknown rx mode - doing nothing"); + case VNET_HW_IF_RX_MODE_DEFAULT: + case VNET_HW_IF_RX_MODE_POLLING: /* fallthrough */ + if (rxq->is_polling) + break; + af_xdp_device_set_rxq_mode (rxq, 1 /* polling */); + break; + case VNET_HW_IF_RX_MODE_INTERRUPT: + case VNET_HW_IF_RX_MODE_ADAPTIVE: /* fallthrough */ + if (0 == rxq->is_polling) + break; + af_xdp_device_set_rxq_mode (rxq, 0 /* interrupt */); + break; + } + + return 0; +} + static void af_xdp_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, u32 node_index) @@ -507,12 +554,12 @@ af_xdp_clear (u32 dev_instance) } /* *INDENT-OFF* */ -VNET_DEVICE_CLASS (af_xdp_device_class) = -{ +VNET_DEVICE_CLASS (af_xdp_device_class) = { .name = "AF_XDP interface", .format_device = format_af_xdp_device, .format_device_name = format_af_xdp_device_name, .admin_up_down_function = af_xdp_interface_admin_up_down, + .rx_mode_change_function = af_xdp_interface_rx_mode_change, .rx_redirect_to_node = af_xdp_set_interface_next_node, .tx_function_n_errors = AF_XDP_TX_N_ERROR, .tx_function_error_strings = af_xdp_tx_func_error_strings, diff --git a/src/plugins/af_xdp/input.c b/src/plugins/af_xdp/input.c index 2cf121fb69d..65eee75cb92 100644 --- a/src/plugins/af_xdp/input.c +++ b/src/plugins/af_xdp/input.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "af_xdp.h" #define foreach_af_xdp_input_error \ @@ -326,22 +327,22 @@ VLIB_NODE_FN (af_xdp_input_node) (vlib_main_t * vm, { u32 n_rx = 0; af_xdp_main_t *am = &af_xdp_main; - vnet_device_input_runtime_t *rt = (void *) node->runtime_data; - vnet_device_and_queue_t *dq; - - foreach_device_and_queue (dq, rt->devices_and_queues) - { - af_xdp_device_t *ad; - ad = vec_elt_at_index (am->devices, dq->dev_instance); - if ((ad->flags & AF_XDP_DEVICE_F_ADMIN_UP) == 0) - continue; - if (PREDICT_TRUE (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY)) - n_rx += af_xdp_device_input_inline (vm, node, frame, ad, dq->queue_id, - /* copy */ 0); - else - n_rx += af_xdp_device_input_inline (vm, node, frame, ad, dq->queue_id, - /* copy */ 1); - } + vnet_hw_if_rxq_poll_vector_t *p, + *pv = vnet_hw_if_get_rxq_poll_vector (vm, node); + + vec_foreach (p, pv) + { + af_xdp_device_t *ad = vec_elt_at_index (am->devices, p->dev_instance); + if ((ad->flags & AF_XDP_DEVICE_F_ADMIN_UP) == 0) + continue; + if (PREDICT_TRUE (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY)) + n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id, + /* copy */ 0); + else + n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id, + /* copy */ 1); + } + return n_rx; } -- 2.16.6