2 /* SPDX-License-Identifier: Apache-2.0
3 * Copyright (c) 2023 Cisco Systems, Inc.
6 #include "vppinfra/bitmap.h"
7 #include "vppinfra/lock.h"
9 #include <vnet/dev/dev.h>
10 #include <vnet/dev/log.h>
12 VLIB_REGISTER_LOG_CLASS (dev_log, static) = {
14 .subclass_name = "runtime",
17 static vnet_dev_rt_op_t *rt_ops;
20 _vnet_dev_rt_exec_op (vlib_main_t *vm, vnet_dev_rt_op_t *op)
22 vnet_dev_port_t *port = op->port;
23 vnet_dev_rx_queue_t *previous = 0, *first = 0;
24 vnet_dev_rx_node_runtime_t *rtd;
25 vlib_node_state_t state = VLIB_NODE_STATE_DISABLED;
26 u32 node_index = port->intf.rx_node_index;
28 rtd = vlib_node_get_runtime_data (vm, node_index);
30 foreach_vnet_dev_port_rx_queue (q, port)
32 if (q->rx_thread_index != vm->thread_index)
35 if (q->interrupt_mode == 0)
36 state = VLIB_NODE_STATE_POLLING;
37 else if (state != VLIB_NODE_STATE_POLLING)
38 state = VLIB_NODE_STATE_INTERRUPT;
40 q->next_on_thread = 0;
44 previous->next_on_thread = q;
49 rtd->first_rx_queue = first;
50 vlib_node_set_state (vm, port->intf.rx_node_index, state);
51 __atomic_store_n (&op->completed, 1, __ATOMIC_RELEASE);
55 vnet_dev_rt_mgmt_node_fn (vlib_main_t *vm, vlib_node_runtime_t *node,
58 u16 thread_index = vm->thread_index;
59 vnet_dev_rt_op_t *op, *ops = __atomic_load_n (&rt_ops, __ATOMIC_ACQUIRE);
65 if (!op->completed && op->thread_index == thread_index)
67 if (op->in_order == 1 && n_pending)
69 vlib_node_set_interrupt_pending (vm, node->node_index);
72 _vnet_dev_rt_exec_op (vm, op);
76 if (op->completed == 0)
83 VLIB_REGISTER_NODE (vnet_dev_rt_mgmt_node, static) = {
84 .function = vnet_dev_rt_mgmt_node_fn,
85 .name = "dev-rt-mgmt",
86 .type = VLIB_NODE_TYPE_PRE_INPUT,
87 .state = VLIB_NODE_STATE_INTERRUPT,
91 vnet_dev_rt_exec_ops (vlib_main_t *vm, vnet_dev_t *dev, vnet_dev_rt_op_t *ops,
94 vnet_dev_rt_op_t *op = ops;
95 vnet_dev_rt_op_t *remote_ops = 0;
96 clib_bitmap_t *remote_bmp = 0;
101 if (vlib_worker_thread_barrier_held ())
103 for (op = ops; op < (ops + n_ops); op++)
105 vlib_main_t *tvm = vlib_get_main_by_index (op->thread_index);
106 _vnet_dev_rt_exec_op (tvm, op);
109 "port %u rx node runtime update on thread %u executed locally",
110 op->port->port_id, op->thread_index);
117 if (op->thread_index != vm->thread_index)
120 _vnet_dev_rt_exec_op (vm, op);
122 dev, "port %u rx node runtime update on thread %u executed locally",
123 op->port->port_id, op->thread_index);
131 for (op = ops; op < (ops + n_ops); op++)
133 if (op->thread_index == vm->thread_index &&
134 (op->in_order == 0 || vec_len (remote_ops) == 0))
136 _vnet_dev_rt_exec_op (vm, op);
138 "port %u rx node runtime update on thread "
139 "%u executed locally",
140 op->port->port_id, op->thread_index);
144 vec_add1 (remote_ops, *op);
146 "port %u rx node runtime update on thread %u "
147 "enqueued for remote execution",
148 op->port->port_id, op->thread_index);
149 remote_bmp = clib_bitmap_set (remote_bmp, op->thread_index, 1);
156 __atomic_store_n (&rt_ops, remote_ops, __ATOMIC_RELEASE);
158 clib_bitmap_foreach (i, remote_bmp)
160 vlib_node_set_interrupt_pending (vlib_get_main_by_index (i),
161 vnet_dev_rt_mgmt_node.index);
162 log_debug (dev, "interrupt sent to %s node on thread %u",
163 vnet_dev_rt_mgmt_node.name, i);
166 vec_foreach (op, remote_ops)
168 while (op->completed == 0)
169 vlib_process_suspend (vm, 5e-5);
172 dev, "port %u rx node runtime update on thread %u executed locally",
173 op->port->port_id, op->thread_index);
176 __atomic_store_n (&rt_ops, 0, __ATOMIC_RELAXED);
177 vec_free (remote_ops);
178 clib_bitmap_free (remote_bmp);