#include <vppinfra/linux/sysfs.h>
#include <vppinfra/unix.h>
#include <vnet/ethernet/ethernet.h>
+#include <vnet/interface/rx_queue_funcs.h>
#include "af_xdp.h"
af_xdp_main_t af_xdp_main;
if (ad->hw_if_index)
{
vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
- vnet_hw_interface_unassign_rx_thread (vnm, ad->hw_if_index, 0);
ethernet_delete_interface (vnm, ad->hw_if_index);
}
af_xdp_create_queue (vlib_main_t * vm, af_xdp_create_if_args_t * args,
af_xdp_device_t * ad, int qid, int rxq_num, int txq_num)
{
- struct xsk_umem **umem = vec_elt_at_index (ad->umem, qid);
- struct xsk_socket **xsk = vec_elt_at_index (ad->xsk, qid);
- af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
- af_xdp_txq_t *txq = vec_elt_at_index (ad->txqs, qid);
+ struct xsk_umem **umem;
+ struct xsk_socket **xsk;
+ af_xdp_rxq_t *rxq;
+ af_xdp_txq_t *txq;
struct xsk_umem_config umem_config;
struct xsk_socket_config sock_config;
struct xdp_options opt;
socklen_t optlen;
+
+ vec_validate_aligned (ad->umem, qid, CLIB_CACHE_LINE_BYTES);
+ umem = vec_elt_at_index (ad->umem, qid);
+
+ vec_validate_aligned (ad->xsk, qid, CLIB_CACHE_LINE_BYTES);
+ xsk = vec_elt_at_index (ad->xsk, qid);
+
+ vec_validate_aligned (ad->rxqs, qid, CLIB_CACHE_LINE_BYTES);
+ rxq = vec_elt_at_index (ad->rxqs, qid);
+
+ vec_validate_aligned (ad->txqs, qid, CLIB_CACHE_LINE_BYTES);
+ txq = vec_elt_at_index (ad->txqs, qid);
+
/*
* fq and cq must always be allocated even if unused
* whereas rx and tx indicates whether we want rxq, txq, or both
static clib_error_t *
af_xdp_device_rxq_read_ready (clib_file_t * f)
{
- vnet_main_t *vnm = vnet_get_main ();
- const af_xdp_main_t *am = &af_xdp_main;
- const u32 dev_instance = f->private_data >> 16;
- const u16 qid = f->private_data & 0xffff;
- const af_xdp_device_t *ad = vec_elt_at_index (am->devices, dev_instance);
- vnet_device_input_set_interrupt_pending (vnm, ad->hw_if_index, qid);
+ vnet_hw_if_rx_queue_set_int_pending (vnet_get_main (), f->private_data);
return 0;
}
+static void
+af_xdp_device_set_rxq_mode (af_xdp_rxq_t *rxq, int is_polling)
+{
+ clib_file_main_t *fm = &file_main;
+ clib_file_t *f;
+
+ if (rxq->is_polling == is_polling)
+ return;
+
+ f = clib_file_get (fm, rxq->file_index);
+ fm->file_update (f, is_polling ? UNIX_FILE_UPDATE_DELETE :
+ UNIX_FILE_UPDATE_ADD);
+ rxq->is_polling = !!is_polling;
+}
+
void
af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
{
goto err1;
q_num = clib_max (rxq_num, txq_num);
- vec_validate_aligned (ad->rxqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
- vec_validate_aligned (ad->txqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
- vec_validate_aligned (ad->umem, q_num - 1, CLIB_CACHE_LINE_BYTES);
- vec_validate_aligned (ad->xsk, q_num - 1, CLIB_CACHE_LINE_BYTES);
ad->txq_num = txq_num;
for (i = 0; i < q_num; i++)
{
/*
* queue creation failed
* it is only a fatal error if we could not create the number of rx
- * queues requested explicitely by the user
+ * queues requested explicitely by the user and the user did not
+ * requested 'max'
* we might create less tx queues than workers but this is ok
*/
- af_xdp_txq_t *txq;
/* fixup vectors length */
vec_set_len (ad->umem, i);
vec_set_len (ad->rxqs, i);
vec_set_len (ad->txqs, i);
- if (i < rxq_num)
+ if (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != rxq_num)
goto err1; /* failed creating requested rxq: fatal error, bailing out */
- /*
- * we created all rxq but failed some txq: not an error but
- * initialize lock for shared txq
- */
- ad->txq_num = i;
- vec_foreach (txq, ad->txqs) clib_spinlock_init (&txq->lock);
+ if (i < txq_num)
+ {
+ /* we created less txq than threads not an error but initialize lock for shared txq */
+ af_xdp_txq_t *txq;
+ ad->txq_num = i;
+ vec_foreach (txq, ad->txqs) clib_spinlock_init (&txq->lock);
+ }
+
args->rv = 0;
clib_error_free (args->error);
break;
args->sw_if_index = ad->sw_if_index = sw->sw_if_index;
hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
- vnet_hw_interface_set_input_node (vnm, ad->hw_if_index,
- af_xdp_input_node.index);
+ vnet_hw_if_set_input_node (vnm, ad->hw_if_index, af_xdp_input_node.index);
- for (i = 0; i < rxq_num; i++)
+ for (i = 0; i < vec_len (ad->rxqs); i++)
{
af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
+ rxq->queue_index = vnet_hw_if_register_rx_queue (
+ vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
+ u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
+ ad->dev_instance, i);
clib_file_t f = {
.file_descriptor = rxq->xsk_fd,
.flags = UNIX_FILE_EVENT_EDGE_TRIGGERED,
- .private_data = (uword) ad->dev_instance << 16 | (uword) i,
+ .private_data = rxq->queue_index,
.read_function = af_xdp_device_rxq_read_ready,
- .description =
- format (0, "%U rxq %d", format_af_xdp_device_name, ad->dev_instance,
- i),
+ .description = desc,
};
rxq->file_index = clib_file_add (&file_main, &f);
- vnet_hw_interface_assign_rx_thread (vnm, ad->hw_if_index, i, ~0);
+ vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
+ rxq->file_index);
+ af_xdp_device_set_rxq_mode (rxq, 1 /* polling */);
}
+ vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
+
/* buffer template */
vec_validate_aligned (ad->buffer_template, 1, CLIB_CACHE_LINE_BYTES);
ad->buffer_template->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
return 0;
}
+static clib_error_t *
+af_xdp_interface_rx_mode_change (vnet_main_t *vnm, u32 hw_if_index, u32 qid,
+ vnet_hw_if_rx_mode mode)
+{
+ af_xdp_main_t *am = &af_xdp_main;
+ vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+ af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
+ af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
+
+ switch (mode)
+ {
+ case VNET_HW_IF_RX_MODE_UNKNOWN:
+ case VNET_HW_IF_NUM_RX_MODES: /* fallthrough */
+ return clib_error_create ("uknown rx mode - doing nothing");
+ case VNET_HW_IF_RX_MODE_DEFAULT:
+ case VNET_HW_IF_RX_MODE_POLLING: /* fallthrough */
+ if (rxq->is_polling)
+ break;
+ af_xdp_device_set_rxq_mode (rxq, 1 /* polling */);
+ break;
+ case VNET_HW_IF_RX_MODE_INTERRUPT:
+ case VNET_HW_IF_RX_MODE_ADAPTIVE: /* fallthrough */
+ if (0 == rxq->is_polling)
+ break;
+ af_xdp_device_set_rxq_mode (rxq, 0 /* interrupt */);
+ break;
+ }
+
+ return 0;
+}
+
static void
af_xdp_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
u32 node_index)
}
/* *INDENT-OFF* */
-VNET_DEVICE_CLASS (af_xdp_device_class) =
-{
+VNET_DEVICE_CLASS (af_xdp_device_class) = {
.name = "AF_XDP interface",
.format_device = format_af_xdp_device,
.format_device_name = format_af_xdp_device_name,
.admin_up_down_function = af_xdp_interface_admin_up_down,
+ .rx_mode_change_function = af_xdp_interface_rx_mode_change,
.rx_redirect_to_node = af_xdp_set_interface_next_node,
.tx_function_n_errors = AF_XDP_TX_N_ERROR,
.tx_function_error_strings = af_xdp_tx_func_error_strings,
Code Review / vpp.git / blobdiff