2 * Copyright (c) 2020 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/vnet.h>
17 #include <vnet/devices/devices.h>
18 #include <vnet/feature/feature.h>
19 #include <vnet/ip/ip.h>
20 #include <vnet/ethernet/ethernet.h>
21 #include <vnet/interface/rx_queue_funcs.h>
22 #include <vnet/interface/tx_queue_funcs.h>
23 #include <vlib/unix/unix.h>
25 VLIB_REGISTER_LOG_CLASS (if_rxq_log, static) = {
26 .class_name = "interface",
27 .subclass_name = "runtime",
30 #define log_debug(fmt, ...) vlib_log_debug (if_rxq_log.class, fmt, __VA_ARGS__)
31 #define log_err(fmt, ...) vlib_log_err (if_rxq_log.class, fmt, __VA_ARGS__)
33 static char *node_state_str[] = {
34 [VLIB_NODE_STATE_DISABLED] = "disabled",
35 [VLIB_NODE_STATE_POLLING] = "polling",
36 [VLIB_NODE_STATE_INTERRUPT] = "interrupt",
40 poll_data_sort (void *a1, void *a2)
42 vnet_hw_if_rxq_poll_vector_t *pv1 = a1;
43 vnet_hw_if_rxq_poll_vector_t *pv2 = a2;
45 if (pv1->dev_instance > pv2->dev_instance)
47 else if (pv1->dev_instance < pv2->dev_instance)
49 else if (pv1->queue_id > pv2->queue_id)
51 else if (pv1->queue_id < pv2->queue_id)
58 vnet_hw_if_update_runtime_data (vnet_main_t *vnm, u32 hw_if_index)
60 vlib_main_t *vm = vlib_get_main ();
61 vnet_interface_main_t *im = &vnm->interface_main;
62 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
63 u32 node_index = hi->input_node_index;
64 vnet_hw_if_rx_queue_t *rxq;
65 vnet_hw_if_rxq_poll_vector_t *pv, **d = 0, **a = 0;
66 vnet_hw_if_output_node_runtime_t *new_out_runtimes = 0;
67 vlib_node_state_t *per_thread_node_state = 0;
68 u32 n_threads = vlib_get_n_threads ();
69 u16 *per_thread_node_adaptive = 0;
70 int something_changed_on_rx = 0;
71 int something_changed_on_tx = 0;
72 clib_bitmap_t *pending_int = 0;
75 log_debug ("update node '%U' triggered by interface %v",
76 format_vlib_node_name, vm, node_index, hi->name);
78 vec_validate (d, n_threads - 1);
79 vec_validate (a, n_threads - 1);
80 vec_validate_init_empty (per_thread_node_state, n_threads - 1,
81 VLIB_NODE_STATE_DISABLED);
82 vec_validate_init_empty (per_thread_node_adaptive, n_threads - 1, 0);
84 /* find out desired node state on each thread */
85 pool_foreach (rxq, im->hw_if_rx_queues)
87 u32 ti = rxq->thread_index;
88 vnet_hw_interface_t *rxq_hi;
90 ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_UNKNOWN);
91 ASSERT (rxq->mode != VNET_HW_IF_RX_MODE_DEFAULT);
93 rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
95 if (rxq_hi->input_node_index != node_index)
98 if (rxq->mode == VNET_HW_IF_RX_MODE_POLLING)
100 per_thread_node_state[ti] = VLIB_NODE_STATE_POLLING;
101 per_thread_node_adaptive[ti] = 0;
104 if (per_thread_node_state[ti] == VLIB_NODE_STATE_POLLING)
107 if (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT ||
108 rxq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)
109 per_thread_node_state[ti] = VLIB_NODE_STATE_INTERRUPT;
111 if (rxq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)
112 per_thread_node_adaptive[ti] = 1;
115 /* construct per-thread polling vectors */
116 pool_foreach (rxq, im->hw_if_rx_queues)
118 u32 ti = rxq->thread_index;
119 vnet_hw_interface_t *rxq_hi;
121 rxq_hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
123 if (rxq_hi->input_node_index != node_index)
126 if (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT ||
127 rxq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE)
128 last_int = clib_max (last_int, rxq - im->hw_if_rx_queues);
130 if (per_thread_node_adaptive[ti])
132 vec_add2_aligned (a[ti], pv, 1, CLIB_CACHE_LINE_BYTES);
133 pv->dev_instance = rxq->dev_instance;
134 pv->queue_id = rxq->queue_id;
137 if (per_thread_node_state[ti] != VLIB_NODE_STATE_POLLING)
140 vec_add2_aligned (d[ti], pv, 1, CLIB_CACHE_LINE_BYTES);
141 pv->dev_instance = rxq->dev_instance;
142 pv->queue_id = rxq->queue_id;
145 /* sort poll vectors and compare them with active ones to avoid
146 * unnecesary barrier */
147 for (int i = 0; i < n_threads; i++)
149 vlib_main_t *ovm = vlib_get_main_by_index (i);
150 vlib_node_state_t old_state;
151 vec_sort_with_function (d[i], poll_data_sort);
153 old_state = vlib_node_get_state (ovm, node_index);
154 if (per_thread_node_state[i] != old_state)
156 something_changed_on_rx = 1;
157 log_debug ("state changed for node %U on thread %u from %s to %s",
158 format_vlib_node_name, vm, node_index, i,
159 node_state_str[old_state],
160 node_state_str[per_thread_node_state[i]]);
163 /* check if something changed */
164 if (something_changed_on_rx == 0)
166 vnet_hw_if_rx_node_runtime_t *rt;
167 rt = vlib_node_get_runtime_data (ovm, node_index);
168 if (vec_len (rt->rxq_vector_int) != vec_len (d[i]))
169 something_changed_on_rx = 1;
170 else if (memcmp (d[i], rt->rxq_vector_int,
171 vec_len (d[i]) * sizeof (**d)))
172 something_changed_on_rx = 1;
173 if (clib_interrupt_get_n_int (rt->rxq_interrupts) != last_int + 1)
174 something_changed_on_rx = 1;
176 if (something_changed_on_rx == 0 && per_thread_node_adaptive[i])
178 if (vec_len (rt->rxq_vector_poll) != vec_len (a[i]))
179 something_changed_on_rx = 1;
180 else if (memcmp (a[i], rt->rxq_vector_poll,
181 vec_len (a[i]) * sizeof (**a)))
182 something_changed_on_rx = 1;
187 if (vec_len (hi->tx_queue_indices) > 0)
189 new_out_runtimes = vec_dup_aligned (hi->output_node_thread_runtimes,
190 CLIB_CACHE_LINE_BYTES);
191 vec_validate_aligned (new_out_runtimes, n_threads - 1,
192 CLIB_CACHE_LINE_BYTES);
194 for (u32 i = 0; i < vec_len (new_out_runtimes); i++)
196 vnet_hw_if_output_node_runtime_t *rt;
197 rt = vec_elt_at_index (new_out_runtimes, i);
198 u32 n_queues = 0, total_queues = vec_len (hi->tx_queue_indices);
200 rt->lookup_table = 0;
202 for (u32 j = 0; j < total_queues; j++)
204 u32 queue_index = hi->tx_queue_indices[j];
205 vnet_hw_if_tx_frame_t frame = { .shared_queue = 0,
208 vnet_hw_if_tx_queue_t *txq =
209 vnet_hw_if_get_tx_queue (vnm, queue_index);
210 if (!clib_bitmap_get (txq->threads, i))
213 log_debug ("tx queue data changed for interface %v, thread %u "
215 hi->name, i, txq->queue_id);
216 something_changed_on_tx = 1;
218 frame.queue_id = txq->queue_id;
219 frame.shared_queue = txq->shared_queue;
220 vec_add1 (rt->frame, frame);
224 // don't initialize rt->n_queues above
225 if (rt->n_queues != n_queues)
227 something_changed_on_tx = 1;
228 rt->n_queues = n_queues;
231 * It is only used in case of multiple txq.
233 if (rt->n_queues > 0)
235 if (!is_pow2 (n_queues))
236 n_queues = max_pow2 (n_queues);
238 vec_validate_aligned (rt->lookup_table, n_queues - 1,
239 CLIB_CACHE_LINE_BYTES);
241 for (u32 k = 0; k < vec_len (rt->lookup_table); k++)
243 rt->lookup_table[k] = rt->frame[k % rt->n_queues].queue_id;
244 log_debug ("tx queue lookup table changed for interface %v, "
245 "(lookup table [%u]=%u)",
246 hi->name, k, rt->lookup_table[k]);
252 /* interface deleted */
253 something_changed_on_tx = 1;
255 if (something_changed_on_rx || something_changed_on_tx)
259 if (vlib_worker_thread_barrier_held ())
262 log_debug ("%s", "already running under the barrier");
268 vlib_worker_thread_barrier_sync (vm);
270 if (something_changed_on_rx)
272 for (int i = 0; i < n_threads; i++)
274 vlib_main_t *vm = vlib_get_main_by_index (i);
275 vnet_hw_if_rx_node_runtime_t *rt;
276 rt = vlib_node_get_runtime_data (vm, node_index);
277 pv = rt->rxq_vector_int;
278 rt->rxq_vector_int = d[i];
281 if (per_thread_node_adaptive[i])
283 pv = rt->rxq_vector_poll;
284 rt->rxq_vector_poll = a[i];
288 if (rt->rxq_interrupts)
290 void *in = rt->rxq_interrupts;
292 while ((int_num = clib_interrupt_get_next (in, int_num)) !=
295 clib_interrupt_clear (in, int_num);
296 pending_int = clib_bitmap_set (pending_int, int_num, 1);
297 last_int = clib_max (last_int, int_num);
301 vlib_node_set_state (vm, node_index, per_thread_node_state[i]);
302 vlib_node_set_flag (vm, node_index, VLIB_NODE_FLAG_ADAPTIVE_MODE,
303 per_thread_node_adaptive[i]);
306 clib_interrupt_resize (&rt->rxq_interrupts, last_int + 1);
308 clib_interrupt_free (&rt->rxq_interrupts);
311 if (something_changed_on_tx)
313 vnet_hw_if_output_node_runtime_t *t;
314 t = hi->output_node_thread_runtimes;
315 hi->output_node_thread_runtimes = new_out_runtimes;
316 new_out_runtimes = t;
320 vlib_worker_thread_barrier_release (vm);
323 log_debug ("skipping update of node '%U', no changes detected",
324 format_vlib_node_name, vm, node_index);
329 clib_bitmap_foreach (i, pending_int)
331 vnet_hw_if_rx_queue_set_int_pending (vnm, i);
333 clib_bitmap_free (pending_int);
336 for (int i = 0; i < n_threads; i++)
340 if (new_out_runtimes)
342 vec_free (new_out_runtimes[i].frame);
343 vec_free (new_out_runtimes[i].lookup_table);
349 vec_free (per_thread_node_state);
350 vec_free (per_thread_node_adaptive);
351 vec_free (new_out_runtimes);