2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
20 #include <linux/if_link.h>
21 #include <bpf/libbpf.h>
22 #include <vlib/vlib.h>
23 #include <vlib/unix/unix.h>
24 #include <vlib/pci/pci.h>
25 #include <vppinfra/linux/sysfs.h>
26 #include <vppinfra/unix.h>
27 #include <vnet/ethernet/ethernet.h>
28 #include <vnet/interface/rx_queue_funcs.h>
31 af_xdp_main_t af_xdp_main;
40 gdb_af_xdp_get_prod (const struct xsk_ring_prod *prod)
42 gdb_af_xdp_pair_t pair = { *prod->producer, *prod->consumer };
47 gdb_af_xdp_get_cons (const struct xsk_ring_cons * cons)
49 gdb_af_xdp_pair_t pair = { *cons->producer, *cons->consumer };
54 af_xdp_mac_change (vnet_hw_interface_t * hw, const u8 * old, const u8 * new)
56 af_xdp_main_t *am = &af_xdp_main;
57 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
58 errno_t err = memcpy_s (ad->hwaddr, sizeof (ad->hwaddr), new, 6);
60 return clib_error_return_code (0, -err, CLIB_ERROR_ERRNO_VALID,
66 af_xdp_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags)
68 af_xdp_main_t *am = &af_xdp_main;
69 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hw->dev_instance);
74 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set unicast not supported yet");
76 case ETHERNET_INTERFACE_FLAG_ACCEPT_ALL:
77 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad,
78 "set promiscuous not supported yet");
80 case ETHERNET_INTERFACE_FLAG_MTU:
81 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "set mtu not supported yet");
85 af_xdp_log (VLIB_LOG_LEVEL_ERR, ad, "unknown flag %x requested", flags);
90 af_xdp_delete_if (vlib_main_t * vm, af_xdp_device_t * ad)
92 vnet_main_t *vnm = vnet_get_main ();
93 af_xdp_main_t *axm = &af_xdp_main;
94 struct xsk_socket **xsk;
95 struct xsk_umem **umem;
101 vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
102 ethernet_delete_interface (vnm, ad->hw_if_index);
105 vec_foreach (rxq, ad->rxqs) clib_file_del_by_index (&file_main,
107 vec_foreach (txq, ad->txqs) clib_spinlock_free (&txq->lock);
108 vec_foreach (xsk, ad->xsk) xsk_socket__delete (*xsk);
109 vec_foreach (umem, ad->umem) xsk_umem__delete (*umem);
113 bpf_set_link_xdp_fd (ad->linux_ifindex, -1, 0);
114 bpf_object__unload (ad->bpf_obj);
119 vec_free (ad->buffer_template);
122 clib_error_free (ad->error);
123 pool_put (axm->devices, ad);
127 af_xdp_load_program (af_xdp_create_if_args_t * args, af_xdp_device_t * ad)
131 ad->linux_ifindex = if_nametoindex (ad->linux_ifname);
132 if (!ad->linux_ifindex)
134 args->rv = VNET_API_ERROR_INVALID_VALUE;
136 clib_error_return_unix (0, "if_nametoindex(%s) failed",
141 if (bpf_prog_load (args->prog, BPF_PROG_TYPE_XDP, &ad->bpf_obj, &fd))
143 args->rv = VNET_API_ERROR_SYSCALL_ERROR_5;
145 clib_error_return_unix (0, "bpf_prog_load(%s) failed", args->prog);
149 #ifndef XDP_FLAGS_REPLACE
150 #define XDP_FLAGS_REPLACE 0
152 if (bpf_set_link_xdp_fd (ad->linux_ifindex, fd, XDP_FLAGS_REPLACE))
154 args->rv = VNET_API_ERROR_SYSCALL_ERROR_6;
156 clib_error_return_unix (0, "bpf_set_link_xdp_fd(%s) failed",
164 bpf_object__unload (ad->bpf_obj);
167 ad->linux_ifindex = ~0;
172 af_xdp_create_queue (vlib_main_t * vm, af_xdp_create_if_args_t * args,
173 af_xdp_device_t * ad, int qid, int rxq_num, int txq_num)
175 struct xsk_umem **umem;
176 struct xsk_socket **xsk;
179 struct xsk_umem_config umem_config;
180 struct xsk_socket_config sock_config;
181 struct xdp_options opt;
184 vec_validate_aligned (ad->umem, qid, CLIB_CACHE_LINE_BYTES);
185 umem = vec_elt_at_index (ad->umem, qid);
187 vec_validate_aligned (ad->xsk, qid, CLIB_CACHE_LINE_BYTES);
188 xsk = vec_elt_at_index (ad->xsk, qid);
190 vec_validate_aligned (ad->rxqs, qid, CLIB_CACHE_LINE_BYTES);
191 rxq = vec_elt_at_index (ad->rxqs, qid);
193 vec_validate_aligned (ad->txqs, qid, CLIB_CACHE_LINE_BYTES);
194 txq = vec_elt_at_index (ad->txqs, qid);
197 * fq and cq must always be allocated even if unused
198 * whereas rx and tx indicates whether we want rxq, txq, or both
200 struct xsk_ring_cons *rx = qid < rxq_num ? &rxq->rx : 0;
201 struct xsk_ring_prod *fq = &rxq->fq;
202 struct xsk_ring_prod *tx = qid < txq_num ? &txq->tx : 0;
203 struct xsk_ring_cons *cq = &txq->cq;
206 memset (&umem_config, 0, sizeof (umem_config));
207 umem_config.fill_size = args->rxq_size;
208 umem_config.comp_size = args->txq_size;
209 umem_config.frame_size =
210 sizeof (vlib_buffer_t) + vlib_buffer_get_default_data_size (vm);
212 * Note about headroom: for some reasons, there seem to be a discrepency
213 * between 0-copy and copy mode:
214 * - 0-copy: XDP_PACKET_HEADROOM will be added to the user headroom
215 * - copy: nothing is added to the user headroom
216 * We privileged 0-copy and set headroom so that frame_headroom +
217 * XDP_PACKET_HEADROOM == sizeof(vlib_buffer_t), ie data will correctly
218 * point to vlib_buffer_t->data for 0-copy. In copy mode, we have to add
219 * XDP_PACKET_HEADROOM to desc offset during refill.
221 STATIC_ASSERT (sizeof (vlib_buffer_t) >= XDP_PACKET_HEADROOM, "wrong size");
222 umem_config.frame_headroom = sizeof (vlib_buffer_t) - XDP_PACKET_HEADROOM;
223 umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG;
225 (umem, uword_to_pointer (vm->buffer_main->buffer_mem_start, void *),
226 vm->buffer_main->buffer_mem_size, fq, cq, &umem_config))
228 args->rv = VNET_API_ERROR_SYSCALL_ERROR_1;
229 args->error = clib_error_return_unix (0, "xsk_umem__create() failed");
233 memset (&sock_config, 0, sizeof (sock_config));
234 sock_config.rx_size = args->rxq_size;
235 sock_config.tx_size = args->txq_size;
236 sock_config.bind_flags = XDP_USE_NEED_WAKEUP;
239 case AF_XDP_MODE_AUTO:
241 case AF_XDP_MODE_COPY:
242 sock_config.bind_flags |= XDP_COPY;
244 case AF_XDP_MODE_ZERO_COPY:
245 sock_config.bind_flags |= XDP_ZEROCOPY;
248 if (xsk_socket__create
249 (xsk, ad->linux_ifname, qid, *umem, rx, tx, &sock_config))
251 args->rv = VNET_API_ERROR_SYSCALL_ERROR_2;
253 clib_error_return_unix (0,
254 "xsk_socket__create() failed (is linux netdev %s up?)",
259 fd = xsk_socket__fd (*xsk);
260 optlen = sizeof (opt);
261 if (getsockopt (fd, SOL_XDP, XDP_OPTIONS, &opt, &optlen))
263 args->rv = VNET_API_ERROR_SYSCALL_ERROR_3;
265 clib_error_return_unix (0, "getsockopt(XDP_OPTIONS) failed");
268 if (opt.flags & XDP_OPTIONS_ZEROCOPY)
269 ad->flags |= AF_XDP_DEVICE_F_ZEROCOPY;
271 rxq->xsk_fd = qid < rxq_num ? fd : -1;
272 txq->xsk_fd = qid < txq_num ? fd : -1;
277 xsk_socket__delete (*xsk);
279 xsk_umem__delete (*umem);
287 af_xdp_get_numa (const char *ifname)
294 (char *) format (0, "/sys/class/net/%s/device/numa_node%c", ifname, 0);
295 err = clib_sysfs_read (path, "%d", &numa);
299 clib_error_free (err);
304 static clib_error_t *
305 af_xdp_device_rxq_read_ready (clib_file_t * f)
307 vnet_hw_if_rx_queue_set_int_pending (vnet_get_main (), f->private_data);
312 af_xdp_device_set_rxq_mode (af_xdp_rxq_t *rxq, int is_polling)
314 clib_file_main_t *fm = &file_main;
317 if (rxq->is_polling == is_polling)
320 f = clib_file_get (fm, rxq->file_index);
321 fm->file_update (f, is_polling ? UNIX_FILE_UPDATE_DELETE :
322 UNIX_FILE_UPDATE_ADD);
323 rxq->is_polling = !!is_polling;
327 af_xdp_create_if (vlib_main_t * vm, af_xdp_create_if_args_t * args)
329 vnet_main_t *vnm = vnet_get_main ();
330 vlib_thread_main_t *tm = vlib_get_thread_main ();
331 af_xdp_main_t *am = &af_xdp_main;
333 vnet_sw_interface_t *sw;
334 vnet_hw_interface_t *hw;
335 int rxq_num, txq_num, q_num;
338 args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
339 args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
340 rxq_num = args->rxq_num ? args->rxq_num : 1;
341 txq_num = tm->n_vlib_mains;
343 if (!args->linux_ifname)
345 args->rv = VNET_API_ERROR_INVALID_VALUE;
346 args->error = clib_error_return (0, "missing host interface");
350 if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE ||
351 args->rxq_size > 65535 || args->txq_size > 65535 ||
352 !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
354 args->rv = VNET_API_ERROR_INVALID_VALUE;
356 clib_error_return (0,
357 "queue size must be a power of two between %i and 65535",
362 pool_get_zero (am->devices, ad);
364 ad->linux_ifname = (char *) format (0, "%s", args->linux_ifname);
365 vec_validate (ad->linux_ifname, IFNAMSIZ - 1); /* libbpf expects ifname to be at least IFNAMSIZ */
367 if (args->prog && af_xdp_load_program (args, ad))
370 q_num = clib_max (rxq_num, txq_num);
371 ad->txq_num = txq_num;
372 for (i = 0; i < q_num; i++)
374 if (af_xdp_create_queue (vm, args, ad, i, rxq_num, txq_num))
377 * queue creation failed
378 * it is only a fatal error if we could not create the number of rx
379 * queues requested explicitely by the user and the user did not
381 * we might create less tx queues than workers but this is ok
384 /* fixup vectors length */
385 vec_set_len (ad->umem, i);
386 vec_set_len (ad->xsk, i);
387 vec_set_len (ad->rxqs, i);
388 vec_set_len (ad->txqs, i);
390 if (i < rxq_num && AF_XDP_NUM_RX_QUEUES_ALL != rxq_num)
391 goto err1; /* failed creating requested rxq: fatal error, bailing out */
395 /* we created less txq than threads not an error but initialize lock for shared txq */
398 vec_foreach (txq, ad->txqs) clib_spinlock_init (&txq->lock);
402 clib_error_free (args->error);
407 ad->dev_instance = ad - am->devices;
408 ad->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
410 vlib_buffer_pool_get_default_for_numa (vm,
415 (char *) format (0, "%s/%d", ad->linux_ifname, ad->dev_instance);
417 ad->name = (char *) format (0, "%s", args->name);
419 ethernet_mac_address_generate (ad->hwaddr);
421 /* create interface */
422 if (ethernet_register_interface (vnm, af_xdp_device_class.index,
423 ad->dev_instance, ad->hwaddr,
424 &ad->hw_if_index, af_xdp_flag_change))
426 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
428 clib_error_return (0, "ethernet_register_interface() failed");
432 sw = vnet_get_hw_sw_interface (vnm, ad->hw_if_index);
433 hw = vnet_get_hw_interface (vnm, ad->hw_if_index);
434 args->sw_if_index = ad->sw_if_index = sw->sw_if_index;
435 hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE;
437 vnet_hw_if_set_input_node (vnm, ad->hw_if_index, af_xdp_input_node.index);
439 for (i = 0; i < vec_len (ad->rxqs); i++)
441 af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
442 rxq->queue_index = vnet_hw_if_register_rx_queue (
443 vnm, ad->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY);
444 u8 *desc = format (0, "%U rxq %d", format_af_xdp_device_name,
445 ad->dev_instance, i);
447 .file_descriptor = rxq->xsk_fd,
448 .flags = UNIX_FILE_EVENT_EDGE_TRIGGERED,
449 .private_data = rxq->queue_index,
450 .read_function = af_xdp_device_rxq_read_ready,
453 rxq->file_index = clib_file_add (&file_main, &f);
454 vnet_hw_if_set_rx_queue_file_index (vnm, rxq->queue_index,
456 af_xdp_device_set_rxq_mode (rxq, 1 /* polling */);
459 vnet_hw_if_update_runtime_data (vnm, ad->hw_if_index);
461 /* buffer template */
462 vec_validate_aligned (ad->buffer_template, 1, CLIB_CACHE_LINE_BYTES);
463 ad->buffer_template->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
464 ad->buffer_template->ref_count = 1;
465 vnet_buffer (ad->buffer_template)->sw_if_index[VLIB_RX] = ad->sw_if_index;
466 vnet_buffer (ad->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0;
467 ad->buffer_template->buffer_pool_index = ad->pool;
472 af_xdp_delete_if (vm, ad);
474 vlib_log_err (am->log_class, "%U", format_clib_error, args->error);
477 static clib_error_t *
478 af_xdp_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
480 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
481 af_xdp_main_t *am = &af_xdp_main;
482 af_xdp_device_t *ad = vec_elt_at_index (am->devices, hi->dev_instance);
483 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
485 if (ad->flags & AF_XDP_DEVICE_F_ERROR)
486 return clib_error_return (0, "device is in error state");
490 vnet_hw_interface_set_flags (vnm, ad->hw_if_index,
491 VNET_HW_INTERFACE_FLAG_LINK_UP);
492 ad->flags |= AF_XDP_DEVICE_F_ADMIN_UP;
496 vnet_hw_interface_set_flags (vnm, ad->hw_if_index, 0);
497 ad->flags &= ~AF_XDP_DEVICE_F_ADMIN_UP;
502 static clib_error_t *
503 af_xdp_interface_rx_mode_change (vnet_main_t *vnm, u32 hw_if_index, u32 qid,
504 vnet_hw_if_rx_mode mode)
506 af_xdp_main_t *am = &af_xdp_main;
507 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
508 af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
509 af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
513 case VNET_HW_IF_RX_MODE_UNKNOWN:
514 case VNET_HW_IF_NUM_RX_MODES: /* fallthrough */
515 return clib_error_create ("uknown rx mode - doing nothing");
516 case VNET_HW_IF_RX_MODE_DEFAULT:
517 case VNET_HW_IF_RX_MODE_POLLING: /* fallthrough */
520 af_xdp_device_set_rxq_mode (rxq, 1 /* polling */);
522 case VNET_HW_IF_RX_MODE_INTERRUPT:
523 case VNET_HW_IF_RX_MODE_ADAPTIVE: /* fallthrough */
524 if (0 == rxq->is_polling)
526 af_xdp_device_set_rxq_mode (rxq, 0 /* interrupt */);
534 af_xdp_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
537 af_xdp_main_t *am = &af_xdp_main;
538 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
539 af_xdp_device_t *ad = pool_elt_at_index (am->devices, hw->dev_instance);
541 /* Shut off redirection */
542 if (node_index == ~0)
544 ad->per_interface_next_index = node_index;
548 ad->per_interface_next_index =
549 vlib_node_add_next (vlib_get_main (), af_xdp_input_node.index,
553 static char *af_xdp_tx_func_error_strings[] = {
555 foreach_af_xdp_tx_func_error
560 af_xdp_clear (u32 dev_instance)
562 af_xdp_main_t *am = &af_xdp_main;
563 af_xdp_device_t *ad = pool_elt_at_index (am->devices, dev_instance);
564 clib_error_free (ad->error);
568 VNET_DEVICE_CLASS (af_xdp_device_class) = {
569 .name = "AF_XDP interface",
570 .format_device = format_af_xdp_device,
571 .format_device_name = format_af_xdp_device_name,
572 .admin_up_down_function = af_xdp_interface_admin_up_down,
573 .rx_mode_change_function = af_xdp_interface_rx_mode_change,
574 .rx_redirect_to_node = af_xdp_set_interface_next_node,
575 .tx_function_n_errors = AF_XDP_TX_N_ERROR,
576 .tx_function_error_strings = af_xdp_tx_func_error_strings,
577 .mac_addr_change_function = af_xdp_mac_change,
578 .clear_counters = af_xdp_clear,
583 af_xdp_init (vlib_main_t * vm)
585 af_xdp_main_t *am = &af_xdp_main;
587 am->log_class = vlib_log_register_class ("af_xdp", 0);
592 VLIB_INIT_FUNCTION (af_xdp_init);
595 * fd.io coding-style-patch-verification: ON
598 * eval: (c-set-style "gnu")