2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/vnet.h>
17 #include <vnet/devices/devices.h>
18 #include <vnet/feature/feature.h>
19 #include <vnet/ip/ip.h>
20 #include <vnet/ethernet/ethernet.h>
22 vnet_device_main_t vnet_device_main;
25 device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
32 VLIB_REGISTER_NODE (device_input_node) = {
33 .function = device_input_fn,
34 .name = "device-input",
35 .runtime_data_bytes = sizeof (vnet_device_input_runtime_t),
36 .type = VLIB_NODE_TYPE_INPUT,
37 .state = VLIB_NODE_STATE_DISABLED,
38 .n_next_nodes = VNET_DEVICE_INPUT_N_NEXT_NODES,
39 .next_nodes = VNET_DEVICE_INPUT_NEXT_NODES,
42 /* Table defines how much we need to advance current data pointer
43 in the buffer if we shortcut to l3 nodes */
45 const u32 __attribute__((aligned (CLIB_CACHE_LINE_BYTES)))
46 device_input_next_node_advance[((VNET_DEVICE_INPUT_N_NEXT_NODES /
47 CLIB_CACHE_LINE_BYTES) +1) * CLIB_CACHE_LINE_BYTES] =
49 [VNET_DEVICE_INPUT_NEXT_IP4_INPUT] = sizeof (ethernet_header_t),
50 [VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT] = sizeof (ethernet_header_t),
51 [VNET_DEVICE_INPUT_NEXT_IP6_INPUT] = sizeof (ethernet_header_t),
52 [VNET_DEVICE_INPUT_NEXT_MPLS_INPUT] = sizeof (ethernet_header_t),
55 const u32 __attribute__((aligned (CLIB_CACHE_LINE_BYTES)))
56 device_input_next_node_flags[((VNET_DEVICE_INPUT_N_NEXT_NODES /
57 CLIB_CACHE_LINE_BYTES) +1) * CLIB_CACHE_LINE_BYTES] =
59 [VNET_DEVICE_INPUT_NEXT_IP4_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
60 [VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
61 [VNET_DEVICE_INPUT_NEXT_IP6_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
62 [VNET_DEVICE_INPUT_NEXT_MPLS_INPUT] = VNET_BUFFER_F_L3_HDR_OFFSET_VALID,
65 VNET_FEATURE_ARC_INIT (device_input, static) =
67 .arc_name = "device-input",
68 .start_nodes = VNET_FEATURES ("device-input"),
69 .last_in_arc = "ethernet-input",
70 .arc_index_ptr = &feature_main.device_input_feature_arc_index,
73 VNET_FEATURE_INIT (l2_patch, static) = {
74 .arc_name = "device-input",
75 .node_name = "l2-patch",
76 .runs_before = VNET_FEATURES ("ethernet-input"),
79 VNET_FEATURE_INIT (worker_handoff, static) = {
80 .arc_name = "device-input",
81 .node_name = "worker-handoff",
82 .runs_before = VNET_FEATURES ("ethernet-input"),
85 VNET_FEATURE_INIT (span_input, static) = {
86 .arc_name = "device-input",
87 .node_name = "span-input",
88 .runs_before = VNET_FEATURES ("ethernet-input"),
91 VNET_FEATURE_INIT (p2p_ethernet_node, static) = {
92 .arc_name = "device-input",
93 .node_name = "p2p-ethernet-input",
94 .runs_before = VNET_FEATURES ("ethernet-input"),
97 VNET_FEATURE_INIT (ethernet_input, static) = {
98 .arc_name = "device-input",
99 .node_name = "ethernet-input",
100 .runs_before = 0, /* not before any other features */
105 vnet_device_queue_sort (void *a1, void *a2)
107 vnet_device_and_queue_t *dq1 = a1;
108 vnet_device_and_queue_t *dq2 = a2;
110 if (dq1->dev_instance > dq2->dev_instance)
112 else if (dq1->dev_instance < dq2->dev_instance)
114 else if (dq1->queue_id > dq2->queue_id)
116 else if (dq1->queue_id < dq2->queue_id)
123 vnet_device_queue_update (vnet_main_t * vnm, vnet_device_input_runtime_t * rt)
125 vnet_device_and_queue_t *dq;
126 vnet_hw_interface_t *hw;
128 vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
130 vec_foreach (dq, rt->devices_and_queues)
132 hw = vnet_get_hw_interface (vnm, dq->hw_if_index);
133 vec_validate (hw->dq_runtime_index_by_queue, dq->queue_id);
134 hw->dq_runtime_index_by_queue[dq->queue_id] = dq - rt->devices_and_queues;
139 vnet_hw_interface_assign_rx_thread (vnet_main_t * vnm, u32 hw_if_index,
140 u16 queue_id, uword thread_index)
142 vnet_device_main_t *vdm = &vnet_device_main;
143 vlib_main_t *vm, *vm0;
144 vnet_device_input_runtime_t *rt;
145 vnet_device_and_queue_t *dq;
146 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
148 ASSERT (hw->input_node_index > 0);
150 if (vdm->first_worker_thread_index == 0)
153 if (thread_index != 0 &&
154 (thread_index < vdm->first_worker_thread_index ||
155 thread_index > vdm->last_worker_thread_index))
157 thread_index = vdm->next_worker_thread_index++;
158 if (vdm->next_worker_thread_index > vdm->last_worker_thread_index)
159 vdm->next_worker_thread_index = vdm->first_worker_thread_index;
162 vm = vlib_mains[thread_index];
163 vm0 = vlib_get_main ();
165 vlib_worker_thread_barrier_sync (vm0);
167 rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
169 vec_add2 (rt->devices_and_queues, dq, 1);
170 dq->hw_if_index = hw_if_index;
171 dq->dev_instance = hw->dev_instance;
172 dq->queue_id = queue_id;
173 dq->mode = VNET_HW_INTERFACE_RX_MODE_POLLING;
174 rt->enabled_node_state = VLIB_NODE_STATE_POLLING;
176 vnet_device_queue_update (vnm, rt);
177 vec_validate (hw->input_node_thread_index_by_queue, queue_id);
178 vec_validate (hw->rx_mode_by_queue, queue_id);
179 hw->input_node_thread_index_by_queue[queue_id] = thread_index;
180 hw->rx_mode_by_queue[queue_id] = VNET_HW_INTERFACE_RX_MODE_POLLING;
182 vlib_worker_thread_barrier_release (vm0);
184 vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state);
188 vnet_hw_interface_unassign_rx_thread (vnet_main_t * vnm, u32 hw_if_index,
191 vlib_main_t *vm, *vm0;
192 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
193 vnet_device_input_runtime_t *rt;
194 vnet_device_and_queue_t *dq;
195 uword old_thread_index;
196 vnet_hw_interface_rx_mode mode;
198 if (hw->input_node_thread_index_by_queue == 0)
199 return VNET_API_ERROR_INVALID_INTERFACE;
201 if (vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1)
202 return VNET_API_ERROR_INVALID_INTERFACE;
204 old_thread_index = hw->input_node_thread_index_by_queue[queue_id];
206 vm = vlib_mains[old_thread_index];
208 rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
210 vec_foreach (dq, rt->devices_and_queues)
211 if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
217 return VNET_API_ERROR_INVALID_INTERFACE;
221 vm0 = vlib_get_main ();
222 vlib_worker_thread_barrier_sync (vm0);
223 vec_del1 (rt->devices_and_queues, dq - rt->devices_and_queues);
224 vnet_device_queue_update (vnm, rt);
225 hw->rx_mode_by_queue[queue_id] = VNET_HW_INTERFACE_RX_MODE_UNKNOWN;
226 vlib_worker_thread_barrier_release (vm0);
228 if (vec_len (rt->devices_and_queues) == 0)
229 vlib_node_set_state (vm, hw->input_node_index, VLIB_NODE_STATE_DISABLED);
230 else if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
233 * if the deleted interface is polling, we may need to set the node state
234 * to interrupt if there is no more polling interface for this device's
235 * corresponding thread. This is because mixed interfaces
236 * (polling and interrupt), assigned to the same thread, set the
237 * thread to polling prior to the deletion.
239 vec_foreach (dq, rt->devices_and_queues)
241 if (dq->mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
244 rt->enabled_node_state = VLIB_NODE_STATE_INTERRUPT;
245 vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state);
253 vnet_hw_interface_set_rx_mode (vnet_main_t * vnm, u32 hw_if_index,
254 u16 queue_id, vnet_hw_interface_rx_mode mode)
258 vnet_device_and_queue_t *dq;
259 vlib_node_state_t enabled_node_state;
260 ASSERT (mode < VNET_HW_INTERFACE_NUM_RX_MODES);
261 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
262 vnet_device_input_runtime_t *rt;
265 if (mode == VNET_HW_INTERFACE_RX_MODE_DEFAULT)
266 mode = hw->default_rx_mode;
268 if (hw->input_node_thread_index_by_queue == 0 || hw->rx_mode_by_queue == 0)
269 return VNET_API_ERROR_INVALID_INTERFACE;
271 if (hw->rx_mode_by_queue[queue_id] == mode)
274 if (mode != VNET_HW_INTERFACE_RX_MODE_POLLING &&
275 (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE) == 0)
276 return VNET_API_ERROR_UNSUPPORTED;
278 if ((vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) ||
279 (vec_len (hw->rx_mode_by_queue) < queue_id + 1))
280 return VNET_API_ERROR_INVALID_QUEUE;
282 hw->rx_mode_by_queue[queue_id] = mode;
283 thread_index = hw->input_node_thread_index_by_queue[queue_id];
284 vm = vlib_mains[thread_index];
286 rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
288 vec_foreach (dq, rt->devices_and_queues)
290 if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
292 if (dq->mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
297 enabled_node_state = VLIB_NODE_STATE_POLLING;
299 enabled_node_state = VLIB_NODE_STATE_INTERRUPT;
301 if (rt->enabled_node_state != enabled_node_state)
303 rt->enabled_node_state = enabled_node_state;
304 if (vlib_node_get_state (vm, hw->input_node_index) !=
305 VLIB_NODE_STATE_DISABLED)
306 vlib_node_set_state (vm, hw->input_node_index, enabled_node_state);
313 vnet_hw_interface_get_rx_mode (vnet_main_t * vnm, u32 hw_if_index,
314 u16 queue_id, vnet_hw_interface_rx_mode * mode)
318 vnet_device_and_queue_t *dq;
319 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
320 vnet_device_input_runtime_t *rt;
322 if (hw->input_node_thread_index_by_queue == 0)
323 return VNET_API_ERROR_INVALID_INTERFACE;
325 if ((vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) ||
326 (vec_len (hw->rx_mode_by_queue) < queue_id + 1))
327 return VNET_API_ERROR_INVALID_QUEUE;
329 thread_index = hw->input_node_thread_index_by_queue[queue_id];
330 vm = vlib_mains[thread_index];
332 rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
334 vec_foreach (dq, rt->devices_and_queues)
335 if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
341 return VNET_API_ERROR_INVALID_INTERFACE;
346 static clib_error_t *
347 vnet_device_init (vlib_main_t * vm)
349 vnet_device_main_t *vdm = &vnet_device_main;
350 vlib_thread_main_t *tm = vlib_get_thread_main ();
351 vlib_thread_registration_t *tr;
354 vec_validate_aligned (vdm->workers, tm->n_vlib_mains - 1,
355 CLIB_CACHE_LINE_BYTES);
357 p = hash_get_mem (tm->thread_registrations_by_name, "workers");
358 tr = p ? (vlib_thread_registration_t *) p[0] : 0;
359 if (tr && tr->count > 0)
361 vdm->first_worker_thread_index = tr->first_index;
362 vdm->next_worker_thread_index = tr->first_index;
363 vdm->last_worker_thread_index = tr->first_index + tr->count - 1;
368 VLIB_INIT_FUNCTION (vnet_device_init);
371 * fd.io coding-style-patch-verification: ON
374 * eval: (c-set-style "gnu")