2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
20 #include <sys/ioctl.h>
21 #include <linux/ethtool.h>
22 #include <linux/if_link.h>
23 #include <linux/sockios.h>
24 #include <linux/limits.h>
25 #include <bpf/libbpf.h>
26 #include <vlib/vlib.h>
27 #include <vlib/unix/unix.h>
28 #include <vlib/pci/pci.h>
29 #include <vppinfra/linux/netns.h>
30 #include <vppinfra/linux/sysfs.h>
31 #include <vppinfra/unix.h>
32 #include <vnet/ethernet/ethernet.h>
33 #include <vnet/interface/rx_queue_funcs.h>
34 #include <vnet/interface/tx_queue_funcs.h>
37 #ifndef XDP_UMEM_MIN_CHUNK_SIZE
38 #define XDP_UMEM_MIN_CHUNK_SIZE 2048
41 af_xdp_main_t af_xdp_main;
50 gdb_af_xdp_get_prod (const struct xsk_ring_prod *prod)
52 gdb_af_xdp_pair_t pair = { *prod->producer, *prod->consumer };
57 gdb_af_xdp_get_cons (const struct xsk_ring_cons * cons)
59 gdb_af_xdp_pair_t pair = { *cons->producer, *cons->consumer };
64 af_xdp_mac_change (vnet_hw_interface_t * hw, const u8 * old, const u8 * new)
66 af_xdp_main_t *am = &af_xdp_main;
67 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
68 errno_t err = memcpy_s (ad->hwaddr, sizeof (ad->hwaddr), new, 6);
70 return clib_error_return_code (0, -err, CLIB_ERROR_ERRNO_VALID,
76 af_xdp_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hw,
79 af_xdp_main_t *am = &af_xdp_main;
80 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
81 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set mtu not supported yet");
82 return vnet_error (VNET_ERR_UNSUPPORTED, 0);
86 af_xdp_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
88 af_xdp_main_t *am = &af_xdp_main;
89 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
94 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set unicast not supported yet");
96 case ETHERNET_INTERFACE_FLAG_ACCEPT_ALL:
97 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
98 "set promiscuous not supported yet");
102 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "unknown flag %x requested", flags);
107 af_xdp_enter_netns (char *netns, int *fds)
109 *fds = *(fds + 1) = -1;
112 *fds = clib_netns_open (NULL /* self */);
113 if ((*(fds + 1) = clib_netns_open ((u8 *) netns)) == -1)
114 return VNET_API_ERROR_SYSCALL_ERROR_8;
115 if (clib_setns (*(fds + 1)) == -1)
116 return VNET_API_ERROR_SYSCALL_ERROR_9;
122 af_xdp_cleanup_netns (int *fds)
127 if (*(fds + 1) != -1)
130 *fds = *(fds + 1) = -1;
134 af_xdp_exit_netns (char *netns, int *fds)
140 ret = clib_setns (*fds);
142 af_xdp_cleanup_netns (fds);
149 af_xdp_delete_if (vlib_main_t * vm, af_xdp_device_t * ad)
151 vnet_main_t *vnm = vnet_get_main ();
152 af_xdp_main_t *axm = &af_xdp_main;
153 struct xsk_socket **xsk;
154 struct xsk_umem **umem;
159 vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
160 ethernet_delete_interface (vnm, ad->hw_if_index);
163 for (i = 0; i < ad->txq_num; i++)
164 clib_spinlock_free (&vec_elt (ad->txqs, i).lock);
166 vec_foreach (xsk, ad->xsk)
167 xsk_socket__delete (*xsk);
169 vec_foreach (umem, ad->umem)
170 xsk_umem__delete (*umem);
172 for (i = 0; i < ad->rxq_num; i++)
173 clib_file_del_by_index (&file_main, vec_elt (ad->rxqs, i).file_index);
178 af_xdp_enter_netns (ad->netns, ns_fds);
179 bpf_set_link_xdp_fd (ad->linux_ifindex, -1, 0);
180 af_xdp_exit_netns (ad->netns, ns_fds);
182 bpf_object__unload (ad->bpf_obj);
187 vec_free (ad->buffer_template);
191 vec_free (ad->linux_ifname);
192 vec_free (ad->netns);
193 clib_error_free (ad->error);
194 pool_put (axm->devices, ad);
198 af_xdp_load_program (af_xdp_create_if_args_t * args, af_xdp_device_t * ad)
201 struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY };
203 if (setrlimit (RLIMIT_MEMLOCK, &r))
204 af_xdp_log (VLIB_LOG_LEVEL_WARNING, ad,
205 "setrlimit(%s) failed: %s (errno %d)", ad->linux_ifname,
206 strerror (errno), errno);
208 ad->linux_ifindex = if_nametoindex (ad->linux_ifname);
209 if (!ad->linux_ifindex)
211 args->rv = VNET_API_ERROR_INVALID_VALUE;
213 clib_error_return_unix (0, "if_nametoindex(%s) failed",
218 if (bpf_prog_load (args->prog, BPF_PROG_TYPE_XDP, &ad->bpf_obj, &fd))
220 args->rv = VNET_API_ERROR_SYSCALL_ERROR_5;
222 clib_error_return_unix (0, "bpf_prog_load(%s) failed", args->prog);
226 if (bpf_set_link_xdp_fd (ad->linux_ifindex, fd, 0))
228 args->rv = VNET_API_ERROR_SYSCALL_ERROR_6;
230 clib_error_return_unix (0, "bpf_set_link_xdp_fd(%s) failed",
238 bpf_object__unload (ad->bpf_obj);
241 ad->linux_ifindex = ~0;
246 af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args,
247 af_xdp_device_t *ad, int qid)
249 struct xsk_umem **umem;
250 struct xsk_socket **xsk;
253 struct xsk_umem_config umem_config;
254 struct xsk_socket_config sock_config;
255 struct xdp_options opt;
257 const int is_rx = qid < ad->rxq_num;
258 const int is_tx = qid < ad->txq_num;
260 umem = vec_elt_at_index (ad->umem, qid);
261 xsk = vec_elt_at_index (ad->xsk, qid);
262 rxq = vec_elt_at_index (ad->rxqs, qid);
263 txq = vec_elt_at_index (ad->txqs, qid);
266 * fq and cq must always be allocated even if unused
267 * whereas rx and tx indicates whether we want rxq, txq, or both
269 struct xsk_ring_cons *rx = is_rx ? &rxq->rx : 0;
270 struct xsk_ring_prod *fq = &rxq->fq;
271 struct xsk_ring_prod *tx = is_tx ? &txq->tx : 0;
272 struct xsk_ring_cons *cq = &txq->cq;
275 memset (&umem_config, 0, sizeof (umem_config));
276 umem_config.fill_size = args->rxq_size;
277 umem_config.comp_size = args->txq_size;
278 umem_config.frame_size =
279 sizeof (vlib_buffer_t) + vlib_buffer_get_default_data_size (vm);
280 umem_config.frame_headroom = sizeof (vlib_buffer_t);
281 umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG;
283 (umem, uword_to_pointer (vm->buffer_main->buffer_mem_start, void *),
284 vm->buffer_main->buffer_mem_size, fq, cq, &umem_config))
286 uword sys_page_size = clib_mem_get_page_size ();
287 args->rv = VNET_API_ERROR_SYSCALL_ERROR_1;
288 args->error = clib_error_return_unix (0, "xsk_umem__create() failed");
289 /* this should mimic the Linux kernel net/xdp/xdp_umem.c:xdp_umem_reg()
291 if (umem_config.frame_size < XDP_UMEM_MIN_CHUNK_SIZE ||
292 umem_config.frame_size > sys_page_size)
293 args->error = clib_error_return (
295 "(unsupported data-size? (should be between %d and %d))",
296 XDP_UMEM_MIN_CHUNK_SIZE - sizeof (vlib_buffer_t),
297 sys_page_size - sizeof (vlib_buffer_t));
301 memset (&sock_config, 0, sizeof (sock_config));
302 sock_config.rx_size = args->rxq_size;
303 sock_config.tx_size = args->txq_size;
304 sock_config.bind_flags = XDP_USE_NEED_WAKEUP;
307 case AF_XDP_MODE_AUTO:
309 case AF_XDP_MODE_COPY:
310 sock_config.bind_flags |= XDP_COPY;
312 case AF_XDP_MODE_ZERO_COPY:
313 sock_config.bind_flags |= XDP_ZEROCOPY;
316 if (xsk_socket__create
317 (xsk, ad->linux_ifname, qid, *umem, rx, tx, &sock_config))
319 args->rv = VNET_API_ERROR_SYSCALL_ERROR_2;
321 clib_error_return_unix (0,
322 "xsk_socket__create() failed (is linux netdev %s up?)",
327 fd = xsk_socket__fd (*xsk);
328 optlen = sizeof (opt);
332 if (getsockopt (fd, SOL_XDP, XDP_OPTIONS, &opt, &optlen))
334 args->rv = VNET_API_ERROR_SYSCALL_ERROR_3;
336 clib_error_return_unix (0, "getsockopt(XDP_OPTIONS) failed");
339 if (opt.flags & XDP_OPTIONS_ZEROCOPY)
340 ad->flags |= AF_XDP_DEVICE_F_ZEROCOPY;
342 rxq->xsk_fd = is_rx ? fd : -1;
347 clib_spinlock_init (&txq->lock);
348 if (is_rx && (ad->flags & AF_XDP_DEVICE_F_SYSCALL_LOCK))
350 /* This is a shared rx+tx queue and we need to lock before syscalls.
351 * Prior to Linux 5.6 there is a race condition preventing to call
352 * poll() and sendto() concurrently on AF_XDP sockets. This was
353 * fixed with commit 11cc2d21499cabe7e7964389634ed1de3ee91d33
354 * to workaround this issue, we protect the syscalls with a
355 * spinlock. Note that it also prevents to use interrupt mode in
356 * multi workers setup, because in this case the poll() is done in
357 * the framework w/o any possibility to protect it.
359 * https://lore.kernel.org/bpf/BYAPR11MB365382C5DB1E5FCC53242609C1549@BYAPR11MB3653.namprd11.prod.outlook.com/
361 clib_spinlock_init (&rxq->syscall_lock);
362 txq->syscall_lock = rxq->syscall_lock;
373 xsk_socket__delete (*xsk);
375 xsk_umem__delete (*umem);
383 af_xdp_get_numa (const char *ifname)
390 (char *) format (0, "/sys/class/net/%s/device/numa_node%c", ifname, 0);
391 err = clib_sysfs_read (path, "%d", &numa);
395 clib_error_free (err);
401 af_xdp_get_q_count (const char *ifname, int *rxq_num, int *txq_num)
403 struct ethtool_channels ec = { .cmd = ETHTOOL_GCHANNELS };
404 struct ifreq ifr = { .ifr_data = (void *) &ec };
407 *rxq_num = *txq_num = 1;
409 fd = socket (AF_INET, SOCK_DGRAM, 0);
413 snprintf (ifr.ifr_name, sizeof (ifr.ifr_name), "%s", ifname);
414 err = ioctl (fd, SIOCETHTOOL, &ifr);
421 *rxq_num = clib_max (ec.combined_count, ec.rx_count);
422 *txq_num = clib_max (ec.combined_count, ec.tx_count);
425 static clib_error_t *
426 af_xdp_device_rxq_read_ready (clib_file_t * f)
428 vnet_hw_if_rx_queue_set_int_pending (vnet_get_main (), f->private_data);
432 static clib_error_t *
433 af_xdp_device_set_rxq_mode (const af_xdp_device_t *ad, af_xdp_rxq_t *rxq,
434 const af_xdp_rxq_mode_t mode)
436 clib_file_main_t *fm = &file_main;
437 clib_file_update_type_t update;
440 if (rxq->mode == mode)
445 case AF_XDP_RXQ_MODE_POLLING:
446 update = UNIX_FILE_UPDATE_DELETE;
448 case AF_XDP_RXQ_MODE_INTERRUPT:
449 if (ad->flags & AF_XDP_DEVICE_F_SYSCALL_LOCK)
450 return clib_error_create (
451 "kernel workaround incompatible with interrupt mode");
452 update = UNIX_FILE_UPDATE_ADD;
456 return clib_error_create ("unknown rxq mode %i", mode);
459 f = clib_file_get (fm, rxq->file_index);
460 fm->file_update (f, update);
466 af_xdp_find_rxq_for_thread (vnet_main_t *vnm, const af_xdp_device_t *ad,
470 for (i = 0; i < ad->rxq_num; i++)
472 const u32 qid = vec_elt (ad->rxqs, i).queue_index;
473 const u32 tid = vnet_hw_if_get_rx_queue (vnm, qid)->thread_index;
480 static clib_error_t *
481 af_xdp_finalize_queues (vnet_main_t *vnm, af_xdp_device_t *ad,
482 const int n_vlib_mains)
484 clib_error_t *err = 0;
487 for (i = 0; i < ad->rxq_num; i++)
489 af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
490 rxq->queue_index = vnet_hw_if_register_rx_queue (
491 vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
492 u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
493 ad->dev_instance, i);
495 .file_descriptor = rxq->xsk_fd,
496 .private_data = rxq->queue_index,
497 .read_function = af_xdp_device_rxq_read_ready,
500 rxq->file_index = clib_file_add (&file_main, &f);
501 vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
503 err = af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING);
508 for (i = 0; i < ad->txq_num; i++)
509 vec_elt (ad->txqs, i).queue_index =
510 vnet_hw_if_register_tx_queue (vnm, ad->hw_if_index, i);
512 /* We set the rxq and txq of the same queue pair on the same thread
513 * by default to avoid locking because of the syscall lock. */
514 int last_qid = clib_min (ad->rxq_num, ad->txq_num - 1);
515 for (i = 0; i < n_vlib_mains; i++)
517 /* search for the 1st rxq assigned on this thread, if any */
518 u32 qid = af_xdp_find_rxq_for_thread (vnm, ad, i);
519 /* if this rxq is combined with a txq, use it. Otherwise, we'll
520 * assign txq in a round-robin fashion. We start from the 1st txq
521 * not shared with a rxq if possible... */
522 qid = qid < ad->txq_num ? qid : (last_qid++ % ad->txq_num);
523 vnet_hw_if_tx_queue_assign_thread (
524 vnm, vec_elt (ad->txqs, qid).queue_index, i);
527 vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
532 af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
534 vnet_main_t *vnm = vnet_get_main ();
535 vlib_thread_main_t *tm = vlib_get_thread_main ();
536 vnet_eth_interface_registration_t eir = {};
537 af_xdp_main_t *am = &af_xdp_main;
539 vnet_sw_interface_t *sw;
540 int rxq_num, txq_num, q_num;
544 args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
545 args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
546 args->rxq_num = args->rxq_num ? args->rxq_num : 1;
548 if (!args->linux_ifname)
550 args->rv = VNET_API_ERROR_INVALID_VALUE;
551 args->error = clib_error_return (0, "missing host interface");
555 if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE ||
556 args->rxq_size > 65535 || args->txq_size > 65535 ||
557 !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
559 args->rv = VNET_API_ERROR_INVALID_VALUE;
561 clib_error_return (0,
562 "queue size must be a power of two between %i and 65535",
567 ret = af_xdp_enter_netns (args->netns, ns_fds);
571 args->error = clib_error_return (0, "enter netns %s failed, ret %d",
572 args->netns, args->rv);
576 af_xdp_get_q_count (args->linux_ifname, &rxq_num, &txq_num);
577 if (args->rxq_num > rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num)
579 args->rv = VNET_API_ERROR_INVALID_VALUE;
580 args->error = clib_error_create ("too many rxq requested (%d > %d)",
581 args->rxq_num, rxq_num);
584 rxq_num = clib_min (rxq_num, args->rxq_num);
585 txq_num = clib_min (txq_num, tm->n_vlib_mains);
587 pool_get_zero (am->devices, ad);
589 if (tm->n_vlib_mains > 1 &&
590 0 == (args->flags & AF_XDP_CREATE_FLAGS_NO_SYSCALL_LOCK))
591 ad->flags |= AF_XDP_DEVICE_F_SYSCALL_LOCK;
593 ad->linux_ifname = (char *) format (0, "%s", args->linux_ifname);
594 vec_validate (ad->linux_ifname, IFNAMSIZ - 1); /* libbpf expects ifname to be at least IFNAMSIZ */
596 ad->netns = (char *) format (0, "%s", args->netns);
598 if (args->prog && af_xdp_load_program (args, ad))
601 q_num = clib_max (rxq_num, txq_num);
602 ad->rxq_num = rxq_num;
603 ad->txq_num = txq_num;
605 vec_validate_aligned (ad->umem, q_num - 1, CLIB_CACHE_LINE_BYTES);
606 vec_validate_aligned (ad->xsk, q_num - 1, CLIB_CACHE_LINE_BYTES);
607 vec_validate_aligned (ad->rxqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
608 vec_validate_aligned (ad->txqs, q_num - 1, CLIB_CACHE_LINE_BYTES);
610 for (i = 0; i < q_num; i++)
612 if (af_xdp_create_queue (vm, args, ad, i))
615 * queue creation failed
616 * it is only a fatal error if we could not create the number of rx
617 * queues requested explicitely by the user and the user did not
619 * we might create less tx queues than workers but this is ok
621 af_xdp_log (VLIB_LOG_LEVEL_DEBUG, ad,
622 "create interface failed to create queue qid=%d", i);
624 /* fixup vectors length */
625 vec_set_len (ad->umem, i);
626 vec_set_len (ad->xsk, i);
627 vec_set_len (ad->rxqs, i);
628 vec_set_len (ad->txqs, i);
630 ad->rxq_num = clib_min (i, rxq_num);
631 ad->txq_num = clib_min (i, txq_num);
634 (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != args->rxq_num))
636 ad->rxq_num = ad->txq_num = 0;
637 goto err2; /* failed creating requested rxq: fatal error, bailing
643 clib_error_free (args->error);
648 if (af_xdp_exit_netns (args->netns, ns_fds))
650 args->rv = VNET_API_ERROR_SYSCALL_ERROR_10;
651 args->error = clib_error_return (0, "exit netns failed");
655 ad->dev_instance = ad - am->devices;
656 ad->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
658 vlib_buffer_pool_get_default_for_numa (vm,
663 char *ifname = ad->linux_ifname;
664 if (args->netns != NULL && strncmp (args->netns, "pid:", 4) == 0)
667 (char *) format (0, "%s/%u", ifname, atoi (args->netns + 4));
670 ad->name = (char *) format (0, "%s/%d", ifname, ad->dev_instance);
673 ad->name = (char *) format (0, "%s", args->name);
675 ethernet_mac_address_generate (ad->hwaddr);
677 /* create interface */
678 eir.dev_class_index = af_xdp_device_class.index;
679 eir.dev_instance = ad->dev_instance;
680 eir.address = ad->hwaddr;
681 eir.cb.flag_change = af_xdp_flag_change;
682 eir.cb.set_max_frame_size = af_xdp_set_max_frame_size;
683 ad->hw_if_index = vnet_eth_register_interface (vnm, &eir);
685 sw = vnet_get_hw_sw_interface (vnm, ad->hw_if_index);
686 args->sw_if_index = ad->sw_if_index = sw->sw_if_index;
688 vnet_hw_if_set_caps (vnm, ad->hw_if_index, VNET_HW_IF_CAP_INT_MODE);
690 vnet_hw_if_set_input_node (vnm, ad->hw_if_index, af_xdp_input_node.index);
692 args->error = af_xdp_finalize_queues (vnm, ad, tm->n_vlib_mains);
695 args->rv = VNET_API_ERROR_SYSCALL_ERROR_7;
699 /* buffer template */
700 vec_validate_aligned (ad->buffer_template, 1, CLIB_CACHE_LINE_BYTES);
701 ad->buffer_template->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
702 ad->buffer_template->ref_count = 1;
703 vnet_buffer (ad->buffer_template)->sw_if_index[VLIB_RX] = ad->sw_if_index;
704 vnet_buffer (ad->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
705 ad->buffer_template->buffer_pool_index = ad->pool;
710 af_xdp_delete_if (vm, ad);
712 af_xdp_cleanup_netns (ns_fds);
714 vlib_log_err (am->log_class, "%U", format_clib_error, args->error);
717 static clib_error_t *
718 af_xdp_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
720 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
721 af_xdp_main_t *am = &af_xdp_main;
722 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hi->dev_instance);
723 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
725 if (ad->flags & AF_XDP_DEVICE_F_ERROR)
726 return clib_error_return (0, "device is in error state");
730 vnet_hw_interface_set_flags (vnm, ad->hw_if_index,
731 VNET_HW_INTERFACE_FLAG_LINK_UP);
732 ad->flags |= AF_XDP_DEVICE_F_ADMIN_UP;
733 af_xdp_device_input_refill (ad);
737 vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
738 ad->flags &= ~AF_XDP_DEVICE_F_ADMIN_UP;
743 static clib_error_t *
744 af_xdp_interface_rx_mode_change (vnet_main_t *vnm, u32 hw_if_index, u32 qid,
745 vnet_hw_if_rx_mode mode)
747 af_xdp_main_t *am = &af_xdp_main;
748 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
749 af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
750 af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
754 default: /* fallthrough */
755 case VNET_HW_IF_RX_MODE_UNKNOWN: /* fallthrough */
756 case VNET_HW_IF_NUM_RX_MODES:
757 return clib_error_create ("uknown rx mode - doing nothing");
758 case VNET_HW_IF_RX_MODE_DEFAULT: /* fallthrough */
759 case VNET_HW_IF_RX_MODE_POLLING:
760 return af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_POLLING);
761 case VNET_HW_IF_RX_MODE_INTERRUPT: /* fallthrough */
762 case VNET_HW_IF_RX_MODE_ADAPTIVE:
763 return af_xdp_device_set_rxq_mode (ad, rxq, AF_XDP_RXQ_MODE_INTERRUPT);
766 ASSERT (0 && "unreachable");
767 return clib_error_create ("unreachable");
771 af_xdp_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
774 af_xdp_main_t *am = &af_xdp_main;
775 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
776 af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
778 /* Shut off redirection */
779 if (node_index == ~0)
781 ad->per_interface_next_index = node_index;
785 ad->per_interface_next_index =
786 vlib_node_add_next (vlib_get_main (), af_xdp_input_node.index,
790 static char *af_xdp_tx_func_error_strings[] = {
792 foreach_af_xdp_tx_func_error
797 af_xdp_clear (u32 dev_instance)
799 af_xdp_main_t *am = &af_xdp_main;
800 af_xdp_device_t *ad = pool_elt_at_index (am->devices, dev_instance);
801 clib_error_free (ad->error);
805 VNET_DEVICE_CLASS (af_xdp_device_class) = {
806 .name = "AF_XDP interface",
807 .format_device = format_af_xdp_device,
808 .format_device_name = format_af_xdp_device_name,
809 .admin_up_down_function = af_xdp_interface_admin_up_down,
810 .rx_mode_change_function = af_xdp_interface_rx_mode_change,
811 .rx_redirect_to_node = af_xdp_set_interface_next_node,
812 .tx_function_n_errors = AF_XDP_TX_N_ERROR,
813 .tx_function_error_strings = af_xdp_tx_func_error_strings,
814 .mac_addr_change_function = af_xdp_mac_change,
815 .clear_counters = af_xdp_clear,
820 af_xdp_init (vlib_main_t * vm)
822 af_xdp_main_t *am = &af_xdp_main;
824 am->log_class = vlib_log_register_class ("af_xdp", 0);
829 VLIB_INIT_FUNCTION (af_xdp_init);
832 * fd.io coding-style-patch-verification: ON
835 * eval: (c-set-style "gnu")