2 * Copyright (c) 2020 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/vnet.h>
17 #include <vnet/devices/devices.h>
18 #include <vnet/interface/rx_queue_funcs.h>
19 #include <vlib/unix/unix.h>
21 VLIB_REGISTER_LOG_CLASS (if_rxq_log, static) = {
22 .class_name = "interface",
23 .subclass_name = "rx-queue",
26 #define log_debug(fmt, ...) vlib_log_debug (if_rxq_log.class, fmt, __VA_ARGS__)
27 #define log_err(fmt, ...) vlib_log_err (if_rxq_log.class, fmt, __VA_ARGS__)
30 next_thread_index (vnet_main_t *vnm, u32 thread_index)
32 vnet_device_main_t *vdm = &vnet_device_main;
33 if (vdm->first_worker_thread_index == 0)
36 if (thread_index != 0 && (thread_index < vdm->first_worker_thread_index ||
37 thread_index > vdm->last_worker_thread_index))
39 thread_index = vdm->next_worker_thread_index++;
40 if (vdm->next_worker_thread_index > vdm->last_worker_thread_index)
41 vdm->next_worker_thread_index = vdm->first_worker_thread_index;
48 rx_queue_key (u32 hw_if_index, u32 queue_id)
50 return ((u64) hw_if_index << 32) | queue_id;
54 vnet_hw_if_get_rx_queue_index_by_id (vnet_main_t *vnm, u32 hw_if_index,
57 vnet_interface_main_t *im = &vnm->interface_main;
58 u64 key = rx_queue_key (hw_if_index, queue_id);
59 uword *p = hash_get_mem (im->rxq_index_by_hw_if_index_and_queue_id, &key);
64 vnet_hw_if_register_rx_queue (vnet_main_t *vnm, u32 hw_if_index, u32 queue_id,
67 vnet_interface_main_t *im = &vnm->interface_main;
68 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
69 vnet_hw_if_rx_queue_t *rxq;
70 u64 key = rx_queue_key (hw_if_index, queue_id);
73 if (hash_get_mem (im->rxq_index_by_hw_if_index_and_queue_id, &key))
74 clib_panic ("Trying to register already registered queue id (%u) in the "
78 thread_index = next_thread_index (vnm, thread_index);
80 pool_get_zero (im->hw_if_rx_queues, rxq);
81 queue_index = rxq - im->hw_if_rx_queues;
82 vec_add1 (hi->rx_queue_indices, queue_index);
83 hash_set_mem_alloc (&im->rxq_index_by_hw_if_index_and_queue_id, &key,
85 rxq->hw_if_index = hw_if_index;
86 rxq->dev_instance = hi->dev_instance;
87 rxq->queue_id = queue_id;
88 rxq->thread_index = thread_index;
89 rxq->mode = VNET_HW_IF_RX_MODE_POLLING;
92 log_debug ("register: interface %v queue-id %u thread %u", hi->name,
93 queue_id, thread_index);
99 vnet_hw_if_unregister_rx_queue (vnet_main_t *vnm, u32 queue_index)
101 vnet_interface_main_t *im = &vnm->interface_main;
102 vnet_hw_if_rx_queue_t *rxq;
103 rxq = vnet_hw_if_get_rx_queue (vnm, queue_index);
104 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
107 key = rx_queue_key (rxq->hw_if_index, rxq->queue_id);
108 hash_unset_mem_free (&im->rxq_index_by_hw_if_index_and_queue_id, &key);
110 for (int i = 0; i < vec_len (hi->rx_queue_indices); i++)
111 if (hi->rx_queue_indices[i] == queue_index)
113 vec_del1 (hi->rx_queue_indices, i);
117 log_debug ("unregister: interface %v queue-id %u", hi->name, rxq->queue_id);
118 pool_put_index (im->hw_if_rx_queues, queue_index);
122 vnet_hw_if_unregister_all_rx_queues (vnet_main_t *vnm, u32 hw_if_index)
124 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
125 vnet_interface_main_t *im = &vnm->interface_main;
126 vnet_hw_if_rx_queue_t *rxq;
128 vnet_hw_if_rx_node_runtime_t *rt;
132 log_debug ("unregister_all: interface %v", hi->name);
134 for (int i = 0; i < vec_len (hi->rx_queue_indices); i++)
136 rxq = vnet_hw_if_get_rx_queue (vnm, hi->rx_queue_indices[i]);
137 key = rx_queue_key (rxq->hw_if_index, rxq->queue_id);
138 if (PREDICT_FALSE (rxq->mode == VNET_HW_IF_RX_MODE_INTERRUPT ||
139 rxq->mode == VNET_HW_IF_RX_MODE_ADAPTIVE))
141 vm = vlib_get_main_by_index (rxq->thread_index);
142 queue_index = vnet_hw_if_get_rx_queue_index_by_id (vnm, hw_if_index,
144 rt = vlib_node_get_runtime_data (vm, hi->input_node_index);
145 clib_interrupt_clear (rt->rxq_interrupts, queue_index);
147 hash_unset_mem_free (&im->rxq_index_by_hw_if_index_and_queue_id, &key);
149 pool_put_index (im->hw_if_rx_queues, hi->rx_queue_indices[i]);
152 vec_free (hi->rx_queue_indices);
156 vnet_hw_if_set_rx_queue_file_index (vnet_main_t *vnm, u32 queue_index,
159 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, queue_index);
160 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
162 rxq->file_index = file_index;
163 clib_file_set_polling_thread (&file_main, file_index, rxq->thread_index);
164 log_debug ("set_file_index: interface %v queue-id %u file-index %u",
165 hi->name, rxq->queue_id, file_index);
169 vnet_hw_if_set_input_node (vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
171 vlib_main_t *vm = vlib_get_main ();
172 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
173 hi->input_node_index = node_index;
174 log_debug ("set_input_node: node %U for interface %v", format_vlib_node_name,
175 vm, node_index, hi->name);
179 vnet_hw_if_set_rx_queue_mode (vnet_main_t *vnm, u32 queue_index,
180 vnet_hw_if_rx_mode mode)
182 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, queue_index);
183 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
184 vnet_device_class_t *dc = vnet_get_device_class (vnm, hi->dev_class_index);
186 ASSERT (mode != VNET_HW_IF_RX_MODE_UNKNOWN);
188 if (mode == VNET_HW_IF_RX_MODE_DEFAULT)
189 mode = hi->default_rx_mode;
191 if (rxq->mode == mode)
193 log_debug ("set_rx_queue_mode: interface %v queue-id %u mode "
195 hi->name, rxq->queue_id, format_vnet_hw_if_rx_mode, mode);
199 if (dc->rx_mode_change_function)
201 clib_error_t *err = dc->rx_mode_change_function (vnm, rxq->hw_if_index,
202 rxq->queue_id, mode);
205 log_err ("setting rx mode on the interface %v queue-id %u failed.\n"
207 hi->name, rxq->queue_id, format_clib_error, err);
208 clib_error_free (err);
209 return VNET_API_ERROR_UNSUPPORTED;
214 log_debug ("set_rx_queue_mode: interface %v queue-id %u mode set to %U",
215 hi->name, rxq->queue_id, format_vnet_hw_if_rx_mode, mode);
220 vnet_hw_if_get_rx_queue_mode (vnet_main_t *vnm, u32 queue_index)
222 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, queue_index);
227 vnet_hw_if_set_rx_queue_thread_index (vnet_main_t *vnm, u32 queue_index,
230 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, queue_index);
231 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, rxq->hw_if_index);
233 rxq->thread_index = thread_index;
235 if (rxq->file_index != ~0)
236 clib_file_set_polling_thread (&file_main, rxq->file_index, thread_index);
238 log_debug ("set_rx_queue_thread_index: interface %v queue-id %u "
239 "thread-index set to %u",
240 hi->name, rxq->queue_id, thread_index);
243 vnet_hw_if_rxq_poll_vector_t *
244 vnet_hw_if_generate_rxq_int_poll_vector (vlib_main_t *vm,
245 vlib_node_runtime_t *node)
247 vnet_hw_if_rx_node_runtime_t *rt = (void *) node->runtime_data;
248 vnet_main_t *vnm = vnet_get_main ();
251 ASSERT (node->state == VLIB_NODE_STATE_INTERRUPT);
253 vec_reset_length (rt->rxq_vector_int);
255 while ((int_num = clib_interrupt_get_next_and_clear (rt->rxq_interrupts,
258 vnet_hw_if_rx_queue_t *rxq = vnet_hw_if_get_rx_queue (vnm, int_num);
259 vnet_hw_if_rxq_poll_vector_t *pv;
261 vec_add2 (rt->rxq_vector_int, pv, 1);
262 pv->dev_instance = rxq->dev_instance;
263 pv->queue_id = rxq->queue_id;
265 return rt->rxq_vector_int;
269 * fd.io coding-style-patch-verification: ON
272 * eval: (c-set-style "gnu")