1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright (c) 2023 Cisco Systems, Inc.
6 #include <vnet/ethernet/ethernet.h>
7 #include <vnet/dev/dev.h>
8 #include <vnet/dev/counters.h>
9 #include <vnet/dev/log.h>
11 VLIB_REGISTER_LOG_CLASS (dev_log, static) = {
13 .subclass_name = "port",
17 dummy_input_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
24 VLIB_REGISTER_NODE (port_rx_eth_node) = {
25 .function = dummy_input_fn,
26 .name = "port-rx-eth",
27 .runtime_data_bytes = sizeof (vnet_dev_rx_node_runtime_t),
28 .type = VLIB_NODE_TYPE_INPUT,
29 .state = VLIB_NODE_STATE_DISABLED,
30 .n_next_nodes = VNET_DEV_ETH_RX_PORT_N_NEXTS,
32 #define _(n, s) [VNET_DEV_ETH_RX_PORT_NEXT_##n] = s,
33 foreach_vnet_dev_port_rx_next
38 u16 vnet_dev_default_next_index_by_port_type[] = {
39 [VNET_DEV_PORT_TYPE_ETHERNET] = VNET_DEV_ETH_RX_PORT_NEXT_ETH_INPUT,
42 VNET_FEATURE_ARC_INIT (eth_port_rx, static) = {
43 .arc_name = "port-rx-eth",
44 .start_nodes = VNET_FEATURES ("port-rx-eth"),
45 .last_in_arc = "ethernet-input",
46 .arc_index_ptr = &vnet_dev_main.eth_port_rx_feature_arc_index,
49 VNET_FEATURE_INIT (l2_patch, static) = {
50 .arc_name = "port-rx-eth",
51 .node_name = "l2-patch",
52 .runs_before = VNET_FEATURES ("ethernet-input"),
55 VNET_FEATURE_INIT (worker_handoff, static) = {
56 .arc_name = "port-rx-eth",
57 .node_name = "worker-handoff",
58 .runs_before = VNET_FEATURES ("ethernet-input"),
61 VNET_FEATURE_INIT (span_input, static) = {
62 .arc_name = "port-rx-eth",
63 .node_name = "span-input",
64 .runs_before = VNET_FEATURES ("ethernet-input"),
67 VNET_FEATURE_INIT (p2p_ethernet_node, static) = {
68 .arc_name = "port-rx-eth",
69 .node_name = "p2p-ethernet-input",
70 .runs_before = VNET_FEATURES ("ethernet-input"),
73 VNET_FEATURE_INIT (ethernet_input, static) = {
74 .arc_name = "port-rx-eth",
75 .node_name = "ethernet-input",
76 .runs_before = 0, /* not before any other features */
80 vnet_dev_port_free (vlib_main_t *vm, vnet_dev_port_t *port)
82 vnet_dev_t *dev = port->dev;
84 vnet_dev_port_validate (vm, port);
86 ASSERT (port->started == 0);
88 log_debug (dev, "port %u", port->port_id);
90 if (port->port_ops.free)
91 port->port_ops.free (vm, port);
93 pool_free (port->secondary_hw_addr);
94 pool_free (port->rx_queues);
95 pool_free (port->tx_queues);
96 vnet_dev_arg_free (&port->args);
97 pool_put_index (dev->ports, port->index);
102 vnet_dev_port_update_tx_node_runtime (vlib_main_t *vm, vnet_dev_port_t *port)
104 vnet_dev_port_validate (vm, port);
106 foreach_vnet_dev_port_tx_queue (q, port)
109 clib_bitmap_foreach (ti, q->assigned_threads)
111 vlib_main_t *tvm = vlib_get_main_by_index (ti);
112 vlib_node_runtime_t *nr =
113 vlib_node_get_runtime (tvm, port->intf.tx_node_index);
114 vnet_dev_tx_node_runtime_t *tnr = vnet_dev_get_tx_node_runtime (nr);
115 tnr->hw_if_index = port->intf.hw_if_index;
122 vnet_dev_port_stop (vlib_main_t *vm, vnet_dev_port_t *port)
124 vnet_dev_t *dev = port->dev;
125 vnet_dev_rt_op_t *ops = 0;
126 u16 n_threads = vlib_get_n_threads ();
128 log_debug (dev, "stopping port %u", port->port_id);
130 for (u16 i = 0; i < n_threads; i++)
132 vnet_dev_rt_op_t op = { .thread_index = i, .port = port };
136 vnet_dev_rt_exec_ops (vm, dev, ops, vec_len (ops));
139 port->port_ops.stop (vm, port);
141 foreach_vnet_dev_port_rx_queue (q, port)
144 log_debug (dev, "port %u rx queue %u stopped", port->port_id,
148 foreach_vnet_dev_port_tx_queue (q, port)
151 log_debug (dev, "port %u tx queue %u stopped", port->port_id,
155 log_debug (dev, "port %u stopped", port->port_id);
160 vnet_dev_port_start_all_rx_queues (vlib_main_t *vm, vnet_dev_port_t *port)
162 vnet_dev_rv_t rv = VNET_DEV_OK;
164 vnet_dev_port_validate (vm, port);
166 foreach_vnet_dev_port_rx_queue (q, port)
168 rv = vnet_dev_rx_queue_start (vm, q);
169 if (rv != VNET_DEV_OK)
176 vnet_dev_port_start_all_tx_queues (vlib_main_t *vm, vnet_dev_port_t *port)
178 vnet_dev_rv_t rv = VNET_DEV_OK;
180 vnet_dev_port_validate (vm, port);
182 foreach_vnet_dev_port_tx_queue (q, port)
184 rv = vnet_dev_tx_queue_start (vm, q);
185 if (rv != VNET_DEV_OK)
192 vnet_dev_port_start (vlib_main_t *vm, vnet_dev_port_t *port)
194 u16 n_threads = vlib_get_n_threads ();
195 vnet_dev_t *dev = port->dev;
196 vnet_dev_rt_op_t *ops = 0;
199 vnet_dev_port_validate (vm, port);
201 log_debug (dev, "starting port %u", port->port_id);
203 vnet_dev_port_update_tx_node_runtime (vm, port);
205 if ((rv = port->port_ops.start (vm, port)) != VNET_DEV_OK)
207 vnet_dev_port_stop (vm, port);
211 for (u16 i = 0; i < n_threads; i++)
213 vnet_dev_rt_op_t op = { .thread_index = i, .port = port };
217 vnet_dev_rt_exec_ops (vm, dev, ops, vec_len (ops));
220 foreach_vnet_dev_port_rx_queue (q, port)
223 log_debug (dev, "port %u rx queue %u started", port->port_id,
228 foreach_vnet_dev_port_tx_queue (q, port)
231 log_debug (dev, "port %u tx queue %u started", port->port_id,
237 log_debug (dev, "port %u started", port->port_id);
243 vnet_dev_port_add (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_port_id_t id,
244 vnet_dev_port_add_args_t *args)
246 vnet_dev_port_t **pp, *port;
247 vnet_dev_rv_t rv = VNET_DEV_OK;
249 ASSERT (args->port.attr.type != VNET_DEV_PORT_TYPE_UNKNOWN);
250 ASSERT (args->port.attr.max_supported_rx_frame_size);
253 vnet_dev_alloc_with_data (sizeof (vnet_dev_port_t), args->port.data_size);
254 pool_get (dev->ports, pp);
256 clib_memcpy (vnet_dev_get_port_data (port), args->port.initial_data,
257 args->port.data_size);
259 port->index = pp - dev->ports;
261 port->attr = args->port.attr;
262 port->rx_queue_config = args->rx_queue.config;
263 port->tx_queue_config = args->tx_queue.config;
264 port->rx_queue_ops = args->rx_queue.ops;
265 port->tx_queue_ops = args->tx_queue.ops;
266 port->port_ops = args->port.ops;
267 port->rx_node = *args->rx_node;
268 port->tx_node = *args->tx_node;
270 for (vnet_dev_arg_t *a = args->port.args; a->type != VNET_DEV_ARG_END; a++)
271 vec_add1 (port->args, *a);
273 /* defaults out of port attributes */
274 port->max_rx_frame_size = args->port.attr.max_supported_rx_frame_size;
275 port->primary_hw_addr = args->port.attr.hw_addr;
277 if (port->attr.type == VNET_DEV_PORT_TYPE_ETHERNET)
279 if (port->max_rx_frame_size > 1514 &&
280 port->attr.caps.change_max_rx_frame_size)
281 port->max_rx_frame_size = 1514;
284 if (port->port_ops.alloc)
285 rv = port->port_ops.alloc (vm, port);
287 if (rv == VNET_DEV_OK)
288 port->initialized = 1;
294 vnet_dev_port_cfg_change_req_validate (vlib_main_t *vm, vnet_dev_port_t *port,
295 vnet_dev_port_cfg_change_req_t *req)
298 vnet_dev_hw_addr_t *addr;
306 case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE:
307 if (req->max_rx_frame_size > port->attr.max_supported_rx_frame_size)
308 return VNET_DEV_ERR_INVALID_VALUE;
309 if (req->max_rx_frame_size == port->max_rx_frame_size)
310 return VNET_DEV_ERR_NO_CHANGE;
313 case VNET_DEV_PORT_CFG_PROMISC_MODE:
314 if (req->promisc == port->promisc)
315 return VNET_DEV_ERR_NO_CHANGE;
318 case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR:
319 if (clib_memcmp (&req->addr, &port->primary_hw_addr,
320 sizeof (vnet_dev_hw_addr_t)) == 0)
321 return VNET_DEV_ERR_NO_CHANGE;
324 case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR:
325 pool_foreach (addr, port->secondary_hw_addr)
326 if (clib_memcmp (addr, &req->addr, sizeof (*addr)) == 0)
327 return VNET_DEV_ERR_ALREADY_EXISTS;
330 case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR:
332 pool_foreach (addr, port->secondary_hw_addr)
333 if (clib_memcmp (addr, &req->addr, sizeof (*addr)) == 0)
336 return VNET_DEV_ERR_NO_SUCH_ENTRY;
343 if (port->port_ops.config_change_validate)
345 rv = port->port_ops.config_change_validate (vm, port, req);
346 if (rv != VNET_DEV_OK)
350 return VNET_DEV_ERR_NOT_SUPPORTED;
357 vnet_dev_port_cfg_change (vlib_main_t *vm, vnet_dev_port_t *port,
358 vnet_dev_port_cfg_change_req_t *req)
360 vnet_dev_rv_t rv = VNET_DEV_OK;
361 vnet_dev_hw_addr_t *a;
362 vnet_dev_rx_queue_t *rxq = 0;
365 vnet_dev_port_validate (vm, port);
367 if (req->type == VNET_DEV_PORT_CFG_RXQ_INTR_MODE_ENABLE ||
368 req->type == VNET_DEV_PORT_CFG_RXQ_INTR_MODE_DISABLE)
370 if (req->all_queues == 0)
372 rxq = vnet_dev_port_get_rx_queue_by_id (port, req->queue_id);
374 return VNET_DEV_ERR_BUG;
378 if ((rv = vnet_dev_port_cfg_change_req_validate (vm, port, req)))
381 if (port->port_ops.config_change)
382 rv = port->port_ops.config_change (vm, port, req);
384 return VNET_DEV_ERR_NOT_SUPPORTED;
386 if (rv != VNET_DEV_OK)
391 case VNET_DEV_PORT_CFG_MAX_RX_FRAME_SIZE:
392 port->max_rx_frame_size = req->max_rx_frame_size;
395 case VNET_DEV_PORT_CFG_PROMISC_MODE:
396 port->promisc = req->promisc;
399 case VNET_DEV_PORT_CFG_RXQ_INTR_MODE_ENABLE:
401 case VNET_DEV_PORT_CFG_RXQ_INTR_MODE_DISABLE:
404 clib_bitmap_t *bmp = 0;
405 vnet_dev_rt_op_t *ops = 0;
408 foreach_vnet_dev_port_rx_queue (q, port)
410 q->interrupt_mode = enable;
411 bmp = clib_bitmap_set (bmp, q->rx_thread_index, 1);
414 clib_bitmap_foreach (i, bmp)
416 vnet_dev_rt_op_t op = { .port = port, .thread_index = i };
420 vnet_dev_rt_exec_ops (vm, port->dev, ops, vec_len (ops));
421 clib_bitmap_free (bmp);
426 rxq->interrupt_mode = enable;
427 vnet_dev_rt_exec_ops (vm, port->dev,
430 .thread_index = rxq->rx_thread_index,
436 case VNET_DEV_PORT_CFG_CHANGE_PRIMARY_HW_ADDR:
437 clib_memcpy (&port->primary_hw_addr, &req->addr,
438 sizeof (vnet_dev_hw_addr_t));
441 case VNET_DEV_PORT_CFG_ADD_SECONDARY_HW_ADDR:
442 pool_get (port->secondary_hw_addr, a);
443 clib_memcpy (a, &req->addr, sizeof (vnet_dev_hw_addr_t));
446 case VNET_DEV_PORT_CFG_REMOVE_SECONDARY_HW_ADDR:
447 pool_foreach (a, port->secondary_hw_addr)
448 if (clib_memcmp (a, &req->addr, sizeof (vnet_dev_hw_addr_t)) == 0)
450 pool_put (port->secondary_hw_addr, a);
463 vnet_dev_port_state_change (vlib_main_t *vm, vnet_dev_port_t *port,
464 vnet_dev_port_state_changes_t changes)
466 vnet_main_t *vnm = vnet_get_main ();
468 vnet_dev_port_validate (vm, port);
470 if (changes.change.link_speed)
472 port->speed = changes.link_speed;
473 if (port->interface_created)
474 vnet_hw_interface_set_link_speed (vnm, port->intf.hw_if_index,
476 log_debug (port->dev, "port speed changed to %u", changes.link_speed);
479 if (changes.change.link_state)
481 port->link_up = changes.link_state;
482 if (port->interface_created)
483 vnet_hw_interface_set_flags (
484 vnm, port->intf.hw_if_index,
485 changes.link_state ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
486 log_debug (port->dev, "port link state changed to %s",
487 changes.link_state ? "up" : "down");
492 vnet_dev_port_add_counters (vlib_main_t *vm, vnet_dev_port_t *port,
493 vnet_dev_counter_t *counters, u16 n_counters)
495 vnet_dev_port_validate (vm, port);
498 vnet_dev_counters_alloc (vm, counters, n_counters, "%s port %u counters",
499 port->dev->device_id, port->port_id);
503 vnet_dev_port_free_counters (vlib_main_t *vm, vnet_dev_port_t *port)
505 vnet_dev_port_validate (vm, port);
507 if (port->counter_main)
508 vnet_dev_counters_free (vm, port->counter_main);
512 vnet_dev_port_if_create (vlib_main_t *vm, vnet_dev_port_t *port)
514 vnet_main_t *vnm = vnet_get_main ();
515 u16 n_threads = vlib_get_n_threads ();
516 vnet_dev_main_t *dm = &vnet_dev_main;
517 vnet_dev_t *dev = port->dev;
518 vnet_dev_port_t **pp;
522 if (port->intf.name[0] == 0)
525 s = format (0, "%s%u/%u",
526 dm->drivers[port->dev->driver_index].registration->name,
527 port->dev->index, port->index);
530 if (n >= sizeof (port->intf.name))
533 return VNET_DEV_ERR_BUG;
535 clib_memcpy (port->intf.name, s, n);
536 port->intf.name[n] = 0;
541 dev, "allocating %u rx queues with size %u and %u tx queues with size %u",
542 port->intf.num_rx_queues, port->intf.rxq_sz, port->intf.num_tx_queues,
545 for (int i = 0; i < port->intf.num_rx_queues; i++)
546 if ((rv = vnet_dev_rx_queue_alloc (vm, port, port->intf.rxq_sz)) !=
550 for (u32 i = 0; i < port->intf.num_tx_queues; i++)
551 if ((rv = vnet_dev_tx_queue_alloc (vm, port, port->intf.txq_sz)) !=
555 foreach_vnet_dev_port_tx_queue (q, port)
557 q->assigned_threads = clib_bitmap_set (q->assigned_threads, ti, 1);
558 log_debug (dev, "port %u tx queue %u assigned to thread %u",
559 port->port_id, q->queue_id, ti);
560 if (++ti >= n_threads)
564 /* pool of port pointers helps us to assign unique dev_instance */
565 pool_get (dm->ports_by_dev_instance, pp);
566 port->intf.dev_instance = pp - dm->ports_by_dev_instance;
569 if (port->attr.type == VNET_DEV_PORT_TYPE_ETHERNET)
571 vnet_device_class_t *dev_class;
572 vnet_dev_driver_t *driver;
573 vnet_sw_interface_t *sw;
574 vnet_hw_interface_t *hw;
577 driver = pool_elt_at_index (dm->drivers, dev->driver_index);
579 /* hack to provide per-port tx node function */
580 dev_class = vnet_get_device_class (vnm, driver->dev_class_index);
581 dev_class->tx_fn_registrations = port->tx_node.registrations;
582 dev_class->format_tx_trace = port->tx_node.format_trace;
583 dev_class->tx_function_error_counters = port->tx_node.error_counters;
584 dev_class->tx_function_n_errors = port->tx_node.n_error_counters;
586 /* create new interface including tx and output nodes */
587 port->intf.hw_if_index = vnet_eth_register_interface (
588 vnm, &(vnet_eth_interface_registration_t){
589 .address = port->primary_hw_addr.eth_mac,
590 .max_frame_size = port->max_rx_frame_size,
591 .dev_class_index = driver->dev_class_index,
592 .dev_instance = port->intf.dev_instance,
593 .cb.set_max_frame_size = vnet_dev_port_set_max_frame_size,
594 .cb.flag_change = vnet_dev_port_eth_flag_change,
597 sw = vnet_get_hw_sw_interface (vnm, port->intf.hw_if_index);
598 hw = vnet_get_hw_interface (vnm, port->intf.hw_if_index);
599 port->intf.sw_if_index = sw->sw_if_index;
600 vnet_hw_interface_set_flags (
601 vnm, port->intf.hw_if_index,
602 port->link_up ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0);
604 vnet_hw_interface_set_link_speed (vnm, port->intf.hw_if_index,
607 port->intf.tx_node_index = hw->tx_node_index;
609 /* create / reuse rx node */
610 if (vec_len (dm->free_rx_node_indices))
613 rx_node_index = vec_pop (dm->free_rx_node_indices);
614 vlib_node_rename (vm, rx_node_index, "%s-rx", port->intf.name);
615 n = vlib_get_node (vm, rx_node_index);
616 n->function = vlib_node_get_preferred_node_fn_variant (
617 vm, port->rx_node.registrations);
618 n->format_trace = port->rx_node.format_trace;
619 vlib_register_errors (vm, rx_node_index,
620 port->rx_node.n_error_counters, 0,
621 port->rx_node.error_counters);
625 dev_class->format_tx_trace = port->tx_node.format_trace;
626 dev_class->tx_function_error_counters = port->tx_node.error_counters;
627 dev_class->tx_function_n_errors = port->tx_node.n_error_counters;
628 vlib_node_registration_t rx_node_reg = {
629 .sibling_of = "port-rx-eth",
630 .type = VLIB_NODE_TYPE_INPUT,
631 .state = VLIB_NODE_STATE_DISABLED,
632 .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED,
633 .node_fn_registrations = port->rx_node.registrations,
634 .format_trace = port->rx_node.format_trace,
635 .error_counters = port->rx_node.error_counters,
636 .n_errors = port->rx_node.n_error_counters,
639 vlib_register_node (vm, &rx_node_reg, "%s-rx", port->intf.name);
641 port->rx_node_assigned = 1;
642 port->intf.rx_node_index = rx_node_index;
643 port->intf.rx_next_index =
644 vnet_dev_default_next_index_by_port_type[port->attr.type];
646 vlib_worker_thread_node_runtime_update ();
648 "ethernet interface created, hw_if_index %u sw_if_index %u "
649 "rx_node_index %u tx_node_index %u",
650 port->intf.hw_if_index, port->intf.sw_if_index,
651 port->intf.rx_node_index, port->intf.tx_node_index);
654 port->interface_created = 1;
655 foreach_vnet_dev_port_rx_queue (q, port)
657 vnet_buffer (&q->buffer_template)->sw_if_index[VLIB_RX] =
658 port->intf.sw_if_index;
659 /* poison to catch node not calling runtime update function */
661 q->interrupt_mode = port->intf.default_is_intr_mode;
662 vnet_dev_rx_queue_rt_request (
663 vm, q, (vnet_dev_rx_queue_rt_req_t){ .update_next_index = 1 });
666 vnet_dev_port_update_tx_node_runtime (vm, port);
668 if (port->port_ops.init)
669 rv = port->port_ops.init (vm, port);
672 if (rv != VNET_DEV_OK)
673 vnet_dev_port_if_remove (vm, port);
678 vnet_dev_port_if_remove (vlib_main_t *vm, vnet_dev_port_t *port)
680 vnet_dev_main_t *dm = &vnet_dev_main;
681 vnet_main_t *vnm = vnet_get_main ();
683 vnet_dev_port_validate (vm, port);
686 vnet_dev_port_stop (vm, port);
688 if (port->rx_node_assigned)
690 vlib_node_rename (vm, port->intf.rx_node_index, "deleted-%u",
691 port->intf.rx_node_index);
692 vec_add1 (dm->free_rx_node_indices, port->intf.rx_node_index);
693 port->rx_node_assigned = 0;
696 if (port->interface_created)
698 vlib_worker_thread_barrier_sync (vm);
699 vnet_delete_hw_interface (vnm, port->intf.hw_if_index);
700 vlib_worker_thread_barrier_release (vm);
701 pool_put_index (dm->ports_by_dev_instance, port->intf.dev_instance);
702 port->interface_created = 0;
705 port->intf = (typeof (port->intf)){};
707 if (port->port_ops.deinit)
708 port->port_ops.deinit (vm, port);
710 foreach_vnet_dev_port_tx_queue (q, port)
711 vnet_dev_tx_queue_free (vm, q);
713 foreach_vnet_dev_port_rx_queue (q, port)
714 vnet_dev_rx_queue_free (vm, q);
716 vnet_dev_port_free_counters (vm, port);
718 foreach_vnet_dev_port_args (v, port)
719 vnet_dev_arg_clear_value (v);
724 vnet_dev_port_clear_counters (vlib_main_t *vm, vnet_dev_port_t *port)
726 if (port->counter_main)
727 vnet_dev_counters_clear (vm, port->counter_main);
729 foreach_vnet_dev_port_rx_queue (q, port)
731 vnet_dev_counters_clear (vm, q->counter_main);
733 foreach_vnet_dev_port_tx_queue (q, port)
735 vnet_dev_counters_clear (vm, q->counter_main);
737 log_notice (port->dev, "counters cleared on port %u", port->port_id);