#include <stdio.h>
#include <net/if.h>
+#include <sys/ioctl.h>
+#include <linux/ethtool.h>
#include <linux/if_link.h>
-#include <bpf/libbpf.h>
+#include <linux/sockios.h>
+#include <linux/limits.h>
+#include <bpf/bpf.h>
#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vlib/pci/pci.h>
+#include <vppinfra/linux/netns.h>
#include <vppinfra/linux/sysfs.h>
#include <vppinfra/unix.h>
#include <vnet/ethernet/ethernet.h>
+#include <vnet/interface/rx_queue_funcs.h>
+#include <vnet/interface/tx_queue_funcs.h>
#include "af_xdp.h"
+#ifndef XDP_UMEM_MIN_CHUNK_SIZE
+#define XDP_UMEM_MIN_CHUNK_SIZE 2048
+#endif
+
af_xdp_main_t af_xdp_main;
typedef struct
return 0;
}
+static clib_error_t *
+af_xdp_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hw,
+ u32 frame_size)
+{
+ af_xdp_main_t *am = &af_xdp_main;
+ af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
+ af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set mtu not supported yet");
+ return vnet_error (VNET_ERR_UNSUPPORTED, 0);
+}
+
static u32
af_xdp_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
{
af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
"set promiscuous not supported yet");
return ~0;
- case ETHERNET_INTERFACE_FLAG_MTU:
- af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set mtu not supported yet");
- return ~0;
}
af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "unknown flag %x requested", flags);
return ~0;
}
+int
+af_xdp_enter_netns (char *netns, int *fds)
+{
+ *fds = *(fds + 1) = -1;
+ if (netns != NULL)
+ {
+ *fds = clib_netns_open (NULL /* self */);
+ if ((*(fds + 1) = clib_netns_open ((u8 *) netns)) == -1)
+ return VNET_API_ERROR_SYSCALL_ERROR_8;
+ if (clib_setns (*(fds + 1)) == -1)
+ return VNET_API_ERROR_SYSCALL_ERROR_9;
+ }
+ return 0;
+}
+
+void
+af_xdp_cleanup_netns (int *fds)
+{
+ if (*fds != -1)
+ close (*fds);
+
+ if (*(fds + 1) != -1)
+ close (*(fds + 1));
+
+ *fds = *(fds + 1) = -1;
+}
+
+int
+af_xdp_exit_netns (char *netns, int *fds)
+{
+ int ret = 0;
+ if (netns != NULL)
+ {
+ if (*fds != -1)
+ ret = clib_setns (*fds);
+
+ af_xdp_cleanup_netns (fds);
+ }
+
+ return ret;
+}
+
+static int
+af_xdp_remove_program (af_xdp_device_t *ad)
+{
+ u32 curr_prog_id = 0;
+ int ret;
+ int ns_fds[2];
+
+ af_xdp_enter_netns (ad->netns, ns_fds);
+ ret = bpf_xdp_query_id (ad->linux_ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST,
+ &curr_prog_id);
+ if (ret != 0)
+ {
+ af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "bpf_xdp_query_id failed\n");
+ goto err0;
+ }
+
+ ret = bpf_xdp_detach (ad->linux_ifindex, XDP_FLAGS_UPDATE_IF_NOEXIST, NULL);
+ if (ret != 0)
+ {
+ af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "bpf_xdp_detach failed\n");
+ goto err0;
+ }
+ af_xdp_exit_netns (ad->netns, ns_fds);
+ if (ad->bpf_obj)
+ bpf_object__close (ad->bpf_obj);
+
+ return 0;
+
+err0:
+ af_xdp_exit_netns (ad->netns, ns_fds);
+ return ret;
+}
+
void
af_xdp_delete_if (vlib_main_t * vm, af_xdp_device_t * ad)
{
af_xdp_main_t *axm = &af_xdp_main;
struct xsk_socket **xsk;
struct xsk_umem **umem;
- af_xdp_rxq_t *rxq;
- af_xdp_txq_t *txq;
+ int i;
if (ad->hw_if_index)
{
vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
- vnet_hw_interface_unassign_rx_thread (vnm, ad->hw_if_index, 0);
ethernet_delete_interface (vnm, ad->hw_if_index);
}
- vec_foreach (rxq, ad->rxqs) clib_file_del_by_index (&file_main,
- rxq->file_index);
- vec_foreach (txq, ad->txqs) clib_spinlock_free (&txq->lock);
- vec_foreach (xsk, ad->xsk) xsk_socket__delete (*xsk);
- vec_foreach (umem, ad->umem) xsk_umem__delete (*umem);
+ for (i = 0; i < ad->txq_num; i++)
+ clib_spinlock_free (&vec_elt (ad->txqs, i).lock);
- if (ad->bpf_obj)
- {
- bpf_set_link_xdp_fd (ad->linux_ifindex, -1, 0);
- bpf_object__unload (ad->bpf_obj);
- }
+ vec_foreach (xsk, ad->xsk)
+ xsk_socket__delete (*xsk);
+
+ vec_foreach (umem, ad->umem)
+ xsk_umem__delete (*umem);
+
+ for (i = 0; i < ad->rxq_num; i++)
+ clib_file_del_by_index (&file_main, vec_elt (ad->rxqs, i).file_index);
+
+ if (af_xdp_remove_program (ad) != 0)
+ af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "Error while removing XDP program.\n");
vec_free (ad->xsk);
vec_free (ad->umem);
vec_free (ad->buffer_template);
vec_free (ad->rxqs);
vec_free (ad->txqs);
+ vec_free (ad->name);
+ vec_free (ad->linux_ifname);
+ vec_free (ad->netns);
clib_error_free (ad->error);
pool_put (axm->devices, ad);
}
af_xdp_load_program (af_xdp_create_if_args_t * args, af_xdp_device_t * ad)
{
int fd;
+ struct bpf_program *bpf_prog;
+ struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY };
- ad->linux_ifindex = if_nametoindex (ad->linux_ifname);
- if (!ad->linux_ifindex)
- {
- args->rv = VNET_API_ERROR_INVALID_VALUE;
- args->error =
- clib_error_return_unix (0, "if_nametoindex(%s) failed",
- ad->linux_ifname);
- goto err0;
- }
+ if (setrlimit (RLIMIT_MEMLOCK, &r))
+ af_xdp_log (VLIB_LOG_LEVEL_WARNING, ad,
+ "setrlimit(%s) failed: %s (errno %d)", ad->linux_ifname,
+ strerror (errno), errno);
- if (bpf_prog_load (args->prog, BPF_PROG_TYPE_XDP, &ad->bpf_obj, &fd))
+ ad->bpf_obj = bpf_object__open_file (args->prog, NULL);
+ if (libbpf_get_error (ad->bpf_obj))
{
args->rv = VNET_API_ERROR_SYSCALL_ERROR_5;
- args->error =
- clib_error_return_unix (0, "bpf_prog_load(%s) failed", args->prog);
+ args->error = clib_error_return_unix (
+ 0, "bpf_object__open_file(%s) failed", args->prog);
goto err0;
}
-#ifndef XDP_FLAGS_REPLACE
-#define XDP_FLAGS_REPLACE 0
-#endif
- if (bpf_set_link_xdp_fd (ad->linux_ifindex, fd, XDP_FLAGS_REPLACE))
+ bpf_prog = bpf_object__next_program (ad->bpf_obj, NULL);
+ if (!bpf_prog)
+ goto err1;
+
+ bpf_program__set_type (bpf_prog, BPF_PROG_TYPE_XDP);
+
+ if (bpf_object__load (ad->bpf_obj))
+ goto err1;
+
+ fd = bpf_program__fd (bpf_prog);
+
+ if (bpf_xdp_attach (ad->linux_ifindex, fd, XDP_FLAGS_UPDATE_IF_NOEXIST,
+ NULL))
{
args->rv = VNET_API_ERROR_SYSCALL_ERROR_6;
- args->error =
- clib_error_return_unix (0, "bpf_set_link_xdp_fd(%s) failed",
- ad->linux_ifname);
+ args->error = clib_error_return_unix (0, "bpf_xdp_attach(%s) failed",
+ ad->linux_ifname);
goto err1;
}
return 0;
err1:
- bpf_object__unload (ad->bpf_obj);
+ bpf_object__close (ad->bpf_obj);
ad->bpf_obj = 0;
err0:
- ad->linux_ifindex = ~0;
return -1;
}
static int
-af_xdp_create_queue (vlib_main_t * vm, af_xdp_create_if_args_t * args,
- af_xdp_device_t * ad, int qid, int rxq_num, int txq_num)
+af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args,
+ af_xdp_device_t *ad, int qid)
{
struct xsk_umem **umem;
struct xsk_socket **xsk;
struct xsk_socket_config sock_config;
struct xdp_options opt;
socklen_t optlen;
+ const int is_rx = qid < ad->rxq_num;
+ const int is_tx = qid < ad->txq_num;
- vec_validate_aligned (ad->umem, qid, CLIB_CACHE_LINE_BYTES);
umem = vec_elt_at_index (ad->umem, qid);
-
- vec_validate_aligned (ad->xsk, qid, CLIB_CACHE_LINE_BYTES);
xsk = vec_elt_at_index (ad->xsk, qid);
-
- vec_validate_aligned (ad->rxqs, qid, CLIB_CACHE_LINE_BYTES);
rxq = vec_elt_at_index (ad->rxqs, qid);
-
- vec_validate_aligned (ad->txqs, qid, CLIB_CACHE_LINE_BYTES);
txq = vec_elt_at_index (ad->txqs, qid);
/*
* fq and cq must always be allocated even if unused
* whereas rx and tx indicates whether we want rxq, txq, or both
*/
- struct xsk_ring_cons *rx = qid < rxq_num ? &rxq->rx : 0;
+ struct xsk_ring_cons *rx = is_rx ? &rxq->rx : 0;
struct xsk_ring_prod *fq = &rxq->fq;
- struct xsk_ring_prod *tx = qid < txq_num ? &txq->tx : 0;
+ struct xsk_ring_prod *tx = is_tx ? &txq->tx : 0;
struct xsk_ring_cons *cq = &txq->cq;
int fd;
umem_config.comp_size = args->txq_size;
umem_config.frame_size =
sizeof (vlib_buffer_t) + vlib_buffer_get_default_data_size (vm);
+ umem_config.frame_headroom = sizeof (vlib_buffer_t);
umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG;
if (xsk_umem__create
(umem, uword_to_pointer (vm->buffer_main->buffer_mem_start, void *),
vm->buffer_main->buffer_mem_size, fq, cq, &umem_config))
{
+ uword sys_page_size = clib_mem_get_page_size ();
args->rv = VNET_API_ERROR_SYSCALL_ERROR_1;
args->error = clib_error_return_unix (0, "xsk_umem__create() failed");
+ /* this should mimic the Linux kernel net/xdp/xdp_umem.c:xdp_umem_reg()
+ * check */
+ if (umem_config.frame_size < XDP_UMEM_MIN_CHUNK_SIZE ||
+ umem_config.frame_size > sys_page_size)
+ args->error = clib_error_return (
+ args->error,
+ "(unsupported data-size? (should be between %d and %d))",
+ XDP_UMEM_MIN_CHUNK_SIZE - sizeof (vlib_buffer_t),
+ sys_page_size - sizeof (vlib_buffer_t));
goto err0;
}
sock_config.bind_flags |= XDP_ZEROCOPY;
break;
}
+ if (args->prog)
+ sock_config.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
if (xsk_socket__create
(xsk, ad->linux_ifname, qid, *umem, rx, tx, &sock_config))
{
}
fd = xsk_socket__fd (*xsk);
+ if (args->prog)
+ {
+ struct bpf_map *map =
+ bpf_object__find_map_by_name (ad->bpf_obj, "xsks_map");
+ int ret = xsk_socket__update_xskmap (*xsk, bpf_map__fd (map));
+ if (ret)
+ {
+ args->rv = VNET_API_ERROR_SYSCALL_ERROR_3;
+ args->error = clib_error_return_unix (
+ 0, "xsk_socket__update_xskmap %s qid %d return %d",
+ ad->linux_ifname, qid, ret);
+ goto err2;
+ }
+ }
optlen = sizeof (opt);
+#ifndef SOL_XDP
+#define SOL_XDP 283
+#endif
if (getsockopt (fd, SOL_XDP, XDP_OPTIONS, &opt, &optlen))
{
- args->rv = VNET_API_ERROR_SYSCALL_ERROR_3;
+ args->rv = VNET_API_ERROR_SYSCALL_ERROR_4;
args->error =
clib_error_return_unix (0, "getsockopt(XDP_OPTIONS) failed");
goto err2;
if (opt.flags & XDP_OPTIONS_ZEROCOPY)
ad->flags |= AF_XDP_DEVICE_F_ZEROCOPY;
- rxq->xsk_fd = qid < rxq_num ? fd : -1;
- txq->xsk_fd = qid < txq_num ? fd : -1;
+ rxq->xsk_fd = is_rx ? fd : -1;
+
+ if (is_tx)
+ {
+ txq->xsk_fd = fd;
+ clib_spinlock_init (&txq->lock);
+ if (is_rx && (ad->flags & AF_XDP_DEVICE_F_SYSCALL_LOCK))
+ {
+ /* This is a shared rx+tx queue and we need to lock before syscalls.
+ * Prior to Linux 5.6 there is a race condition preventing to call
+ * poll() and sendto() concurrently on AF_XDP sockets. This was
+ * fixed with commit 11cc2d21499cabe7e7964389634ed1de3ee91d33
+ * to workaround this issue, we protect the syscalls with a
+ * spinlock. Note that it also prevents to use interrupt mode in
+ * multi workers setup, because in this case the poll() is done in
+ * the framework w/o any possibility to protect it.
+ * See
+ * https://lore.kernel.org/bpf/BYAPR11MB365382C5DB1E5FCC53242609C1549@BYAPR11MB3653.namprd11.prod.outlook.com/
+ */
+ clib_spinlock_init (&rxq->syscall_lock);
+ txq->syscall_lock = rxq->syscall_lock;
+ }
+ }
+ else
+ {
+ txq->xsk_fd = -1;
+ }
return 0;
return numa;
}
+static void
+af_xdp_get_q_count (const char *ifname, int *rxq_num, int *txq_num)
+{
+ struct ethtool_channels ec = { .cmd = ETHTOOL_GCHANNELS };
+ struct ifreq ifr = { .ifr_data = (void *) &ec };
+ int fd, err;
+
+ *rxq_num = *txq_num = 1;
+
+ fd = socket (AF_INET, SOCK_DGRAM, 0);
+ if (fd < 0)
+ return;
+
+ snprintf (ifr.ifr_name, sizeof (ifr.ifr_name), "%s", ifname);
+ err = ioctl (fd, SIOCETHTOOL, &ifr);
+
+ close (fd);
+
+ if (err)
+ return;
+
+ *rxq_num = clib_max (ec.combined_count, ec.rx_count);
+ *txq_num = clib_max (ec.combined_count, ec.tx_count);
+}
+
static clib_error_t *
af_xdp_device_rxq_read_ready (clib_file_t * f)
{
- vnet_main_t *vnm = vnet_get_main ();
- const af_xdp_main_t *am = &af_xdp_main;
- const u32 dev_instance = f->private_data >> 16;
- const u16 qid = f->private_data & 0xffff;
- const af_xdp_device_t *ad = vec_elt_at_index (am->devices, dev_instance);
- vnet_device_input_set_interrupt_pending (vnm, ad->hw_if_index, qid);
+ vnet_hw_if_rx_queue_set_int_pending (vnet_get_main (), f->private_data);
+ return 0;
+}
+
+static clib_error_t *
+af_xdp_device_set_rxq_mode (const af_xdp_device_t *ad, af_xdp_rxq_t *rxq,
+ const af_xdp_rxq_mode_t mode)
+{
+ clib_file_main_t *fm = &file_main;
+ clib_file_update_type_t update;
+ clib_file_t *f;
+
+ if (rxq->mode == mode)
+ return 0;
+
+ switch (mode)
+ {
+ case AF_XDP_RXQ_MODE_POLLING:
+ update = UNIX_FILE_UPDATE_DELETE;
+ break;
+ case AF_XDP_RXQ_MODE_INTERRUPT:
+ if (ad->flags & AF_XDP_DEVICE_F_SYSCALL_LOCK)
+ return clib_error_create (
+ "kernel workaround incompatible with interrupt mode");
+ update = UNIX_FILE_UPDATE_ADD;
+ break;
+ default:
+ ASSERT (0);
+ return clib_error_create ("unknown rxq mode %i", mode);
+ }
+
+ f = clib_file_get (fm, rxq->file_index);
+ fm->file_update (f, update);
+ rxq->mode = mode;
+ return 0;
+}
+
+static u32
+af_xdp_find_rxq_for_thread (vnet_main_t *vnm, const af_xdp_device_t *ad,
+ const u32 thread)
+{
+ u32 i;
+ for (i = 0; i < ad->rxq_num; i++)
+ {
+ const u32 qid = vec_elt (ad->rxqs, i).queue_index;
+ const u32 tid = vnet_hw_if_get_rx_queue (vnm, qid)->thread_index;
+ if (tid == thread)
+ return i;
+ }
+ return ~0;
+}
+
+static clib_error_t *
+af_xdp_finalize_queues (vnet_main_t *vnm, af_xdp_device_t *ad,
+ const int n_vlib_mains)
+{
+ clib_error_t *err = 0;
+ int i;
+
+ for (i = 0; i < ad->rxq_num; i++)
+ {
+ af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
+ rxq->queue_index = vnet_hw_if_register_rx_queue (
+ vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
+ u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
+ ad->dev_instance, i);
+ clib_file_t f = {
+ .file_descriptor = rxq->xsk_fd,
+ .private_data = rxq->queue_index,
+ .read_function = af_xdp_device_rxq_read_ready,
+ .description = desc,
+ };
+ rxq->file_index = clib_file_add (&file_main, &f);
+ vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
+ rxq->file_index);
+ err = af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING);
+ if (err)
+ return err;
+ }
+
+ for (i = 0; i < ad->txq_num; i++)
+ vec_elt (ad->txqs, i).queue_index =
+ vnet_hw_if_register_tx_queue (vnm, ad->hw_if_index, i);
+
+ /* We set the rxq and txq of the same queue pair on the same thread
+ * by default to avoid locking because of the syscall lock. */
+ int last_qid = clib_min (ad->rxq_num, ad->txq_num - 1);
+ for (i = 0; i < n_vlib_mains; i++)
+ {
+ /* search for the 1st rxq assigned on this thread, if any */
+ u32 qid = af_xdp_find_rxq_for_thread (vnm, ad, i);
+ /* if this rxq is combined with a txq, use it. Otherwise, we'll
+ * assign txq in a round-robin fashion. We start from the 1st txq
+ * not shared with a rxq if possible... */
+ qid = qid < ad->txq_num ? qid : (last_qid++ % ad->txq_num);
+ vnet_hw_if_tx_queue_assign_thread (
+ vnm, vec_elt (ad->txqs, qid).queue_index, i);
+ }
+
+ vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
return 0;
}
{
vnet_main_t *vnm = vnet_get_main ();
vlib_thread_main_t *tm = vlib_get_thread_main ();
+ vnet_eth_interface_registration_t eir = {};
af_xdp_main_t *am = &af_xdp_main;
af_xdp_device_t *ad;
vnet_sw_interface_t *sw;
- vnet_hw_interface_t *hw;
int rxq_num, txq_num, q_num;
- int i;
+ int ns_fds[2];
+ int i, ret;
args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
- rxq_num = args->rxq_num ? args->rxq_num : 1;
- txq_num = tm->n_vlib_mains;
+ args->rxq_num = args->rxq_num ? args->rxq_num : 1;
if (!args->linux_ifname)
{
goto err0;
}
+ ret = af_xdp_enter_netns (args->netns, ns_fds);
+ if (ret)
+ {
+ args->rv = ret;
+ args->error = clib_error_return (0, "enter netns %s failed, ret %d",
+ args->netns, args->rv);
+ goto err0;
+ }
+
+ af_xdp_get_q_count (args->linux_ifname, &rxq_num, &txq_num);
+ if (args->rxq_num > rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num)
+ {
+ args->rv = VNET_API_ERROR_INVALID_VALUE;
+ args->error = clib_error_create ("too many rxq requested (%d > %d)",
+ args->rxq_num, rxq_num);
+ goto err1;
+ }
+ rxq_num = clib_min (rxq_num, args->rxq_num);
+ txq_num = clib_min (txq_num, tm->n_vlib_mains);
+
pool_get_zero (am->devices, ad);
+ if (tm->n_vlib_mains > 1 &&
+ 0 == (args->flags & AF_XDP_CREATE_FLAGS_NO_SYSCALL_LOCK))
+ ad->flags |= AF_XDP_DEVICE_F_SYSCALL_LOCK;
+
ad->linux_ifname = (char *) format (0, "%s", args->linux_ifname);
vec_validate (ad->linux_ifname, IFNAMSIZ - 1); /* libbpf expects ifname to be at least IFNAMSIZ */
- if (args->prog && af_xdp_load_program (args, ad))
- goto err1;
+ if (args->netns)
+ ad->netns = (char *) format (0, "%s%c", args->netns, 0);
+
+ ad->linux_ifindex = if_nametoindex (ad->linux_ifname);
+ if (!ad->linux_ifindex)
+ {
+ args->rv = VNET_API_ERROR_INVALID_VALUE;
+ args->error = clib_error_return_unix (0, "if_nametoindex(%s) failed",
+ ad->linux_ifname);
+ ad->linux_ifindex = ~0;
+ goto err1;
+ }
+
+ if (args->prog &&
+ (af_xdp_remove_program (ad) || af_xdp_load_program (args, ad)))
+ goto err2;
q_num = clib_max (rxq_num, txq_num);
+ ad->rxq_num = rxq_num;
ad->txq_num = txq_num;
+
+ vec_validate_aligned (ad->umem, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->xsk, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->rxqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
+ vec_validate_aligned (ad->txqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
+
for (i = 0; i < q_num; i++)
{
- if (af_xdp_create_queue (vm, args, ad, i, rxq_num, txq_num))
+ if (af_xdp_create_queue (vm, args, ad, i))
{
/*
* queue creation failed
* requested 'max'
* we might create less tx queues than workers but this is ok
*/
+ af_xdp_log (VLIB_LOG_LEVEL_DEBUG, ad,
+ "create interface failed to create queue qid=%d", i);
/* fixup vectors length */
vec_set_len (ad->umem, i);
vec_set_len (ad->rxqs, i);
vec_set_len (ad->txqs, i);
- if (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != rxq_num)
- goto err1; /* failed creating requested rxq: fatal error, bailing out */
+ ad->rxq_num = clib_min (i, rxq_num);
+ ad->txq_num = clib_min (i, txq_num);
- if (i < txq_num)
+ if (i == 0 ||
+ (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num))
{
- /* we created less txq than threads not an error but initialize lock for shared txq */
- af_xdp_txq_t *txq;
- ad->txq_num = i;
- vec_foreach (txq, ad->txqs) clib_spinlock_init (&txq->lock);
+ ad->rxq_num = ad->txq_num = 0;
+ goto err2; /* failed creating requested rxq: fatal error, bailing
+ out */
}
+
args->rv = 0;
clib_error_free (args->error);
break;
}
}
+ if (af_xdp_exit_netns (args->netns, ns_fds))
+ {
+ args->rv = VNET_API_ERROR_SYSCALL_ERROR_10;
+ args->error = clib_error_return (0, "exit netns failed");
+ goto err2;
+ }
+
ad->dev_instance = ad - am->devices;
ad->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
ad->pool =
af_xdp_get_numa
(ad->linux_ifname));
if (!args->name)
- ad->name =
- (char *) format (0, "%s/%d", ad->linux_ifname, ad->dev_instance);
+ {
+ char *ifname = ad->linux_ifname;
+ if (args->netns != NULL && strncmp (args->netns, "pid:", 4) == 0)
+ {
+ ad->name =
+ (char *) format (0, "%s/%u", ifname, atoi (args->netns + 4));
+ }
+ else
+ ad->name = (char *) format (0, "%s/%d", ifname, ad->dev_instance);
+ }
else
ad->name = (char *) format (0, "%s", args->name);
ethernet_mac_address_generate (ad->hwaddr);
/* create interface */
- if (ethernet_register_interface (vnm, af_xdp_device_class.index,
- ad->dev_instance, ad->hwaddr,
- &ad->hw_if_index, af_xdp_flag_change))
- {
- args->rv = VNET_API_ERROR_INVALID_INTERFACE;
- args->error =
- clib_error_return (0, "ethernet_register_interface() failed");
- goto err1;
- }
+ eir.dev_class_index = af_xdp_device_class.index;
+ eir.dev_instance = ad->dev_instance;
+ eir.address = ad->hwaddr;
+ eir.cb.flag_change = af_xdp_flag_change;
+ eir.cb.set_max_frame_size = af_xdp_set_max_frame_size;
+ ad->hw_if_index = vnet_eth_register_interface (vnm, &eir);
sw = vnet_get_hw_sw_interface (vnm, ad->hw_if_index);
- hw = vnet_get_hw_interface (vnm, ad->hw_if_index);
args->sw_if_index = ad->sw_if_index = sw->sw_if_index;
- hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
- vnet_hw_interface_set_input_node (vnm, ad->hw_if_index,
- af_xdp_input_node.index);
+ vnet_hw_if_set_caps (vnm, ad->hw_if_index, VNET_HW_IF_CAP_INT_MODE);
+
+ vnet_hw_if_set_input_node (vnm, ad->hw_if_index, af_xdp_input_node.index);
- for (i = 0; i < vec_len (ad->rxqs); i++)
+ args->error = af_xdp_finalize_queues (vnm, ad, tm->n_vlib_mains);
+ if (args->error)
{
- af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
- clib_file_t f = {
- .file_descriptor = rxq->xsk_fd,
- .flags = UNIX_FILE_EVENT_EDGE_TRIGGERED,
- .private_data = (uword) ad->dev_instance << 16 | (uword) i,
- .read_function = af_xdp_device_rxq_read_ready,
- .description =
- format (0, "%U rxq %d", format_af_xdp_device_name, ad->dev_instance,
- i),
- };
- rxq->file_index = clib_file_add (&file_main, &f);
- vnet_hw_interface_assign_rx_thread (vnm, ad->hw_if_index, i, ~0);
+ args->rv = VNET_API_ERROR_SYSCALL_ERROR_7;
+ goto err2;
}
/* buffer template */
return;
-err1:
+err2:
af_xdp_delete_if (vm, ad);
+err1:
+ af_xdp_cleanup_netns (ns_fds);
err0:
vlib_log_err (am->log_class, "%U", format_clib_error, args->error);
}
vnet_hw_interface_set_flags (vnm, ad->hw_if_index,
VNET_HW_INTERFACE_FLAG_LINK_UP);
ad->flags |= AF_XDP_DEVICE_F_ADMIN_UP;
+ af_xdp_device_input_refill (ad);
}
else
{
return 0;
}
+static clib_error_t *
+af_xdp_interface_rx_mode_change (vnet_main_t *vnm, u32 hw_if_index, u32 qid,
+ vnet_hw_if_rx_mode mode)
+{
+ af_xdp_main_t *am = &af_xdp_main;
+ vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+ af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
+ af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
+
+ switch (mode)
+ {
+ default: /* fallthrough */
+ case VNET_HW_IF_RX_MODE_UNKNOWN: /* fallthrough */
+ case VNET_HW_IF_NUM_RX_MODES:
+ return clib_error_create ("uknown rx mode - doing nothing");
+ case VNET_HW_IF_RX_MODE_DEFAULT: /* fallthrough */
+ case VNET_HW_IF_RX_MODE_POLLING:
+ return af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING);
+ case VNET_HW_IF_RX_MODE_INTERRUPT: /* fallthrough */
+ case VNET_HW_IF_RX_MODE_ADAPTIVE:
+ return af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_INTERRUPT);
+ }
+
+ ASSERT (0 && "unreachable");
+ return clib_error_create ("unreachable");
+}
+
static void
af_xdp_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
u32 node_index)
clib_error_free (ad->error);
}
-/* *INDENT-OFF* */
-VNET_DEVICE_CLASS (af_xdp_device_class) =
-{
+VNET_DEVICE_CLASS (af_xdp_device_class) = {
.name = "AF_XDP interface",
.format_device = format_af_xdp_device,
.format_device_name = format_af_xdp_device_name,
.admin_up_down_function = af_xdp_interface_admin_up_down,
+ .rx_mode_change_function = af_xdp_interface_rx_mode_change,
.rx_redirect_to_node = af_xdp_set_interface_next_node,
.tx_function_n_errors = AF_XDP_TX_N_ERROR,
.tx_function_error_strings = af_xdp_tx_func_error_strings,
.mac_addr_change_function = af_xdp_mac_change,
.clear_counters = af_xdp_clear,
};
-/* *INDENT-ON* */
clib_error_t *
af_xdp_init (vlib_main_t * vm)
Code Review / vpp.git / blobdiff