2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/error.h>
18 #include <vppinfra/format.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/dpdk/dpdk.h>
24 #include <vlibmemory/api.h>
25 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
27 #define vl_typedefs /* define message structures */
28 #include <vlibmemory/vl_memory_api_h.h>
31 /* instantiate all the print functions we know about */
32 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
34 #include <vlibmemory/vl_memory_api_h.h>
37 vlib_thread_main_t vlib_thread_main;
39 frame_queue_trace_t *frame_queue_traces;
42 * Check the frame queue to see if any frames are available.
43 * If so, pull the packets off the frames and put them to
46 static inline int vlib_frame_queue_dequeue_internal (vlib_main_t *vm)
48 u32 thread_id = vm->cpu_index;
49 vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
50 vlib_frame_queue_elt_t *elt;
59 ASSERT(vm == vlib_mains[thread_id]);
62 * Gather trace data for frame queues
64 if (PREDICT_FALSE(fq->trace))
66 frame_queue_trace_t *fqt;
69 fqt = &frame_queue_traces[thread_id];
70 fqt->nelts = fq->nelts;
72 fqt->head_hint = fq->head_hint;
74 fqt->threshold = fq->vector_threshold;
75 fqt->n_in_use = fqt->tail - fqt->head;
76 if (fqt->n_in_use > fqt->nelts){
80 for (elix=0; elix<fqt->nelts; elix++) {
81 elt = fq->elts + ((fq->head+1 + elix) & (fq->nelts-1));
84 fqt->n_vectors[elix] = elt->n_vectors;
92 if (fq->head == fq->tail)
94 fq->head_hint = fq->head;
98 elt = fq->elts + ((fq->head+1) & (fq->nelts-1));
102 fq->head_hint = fq->head;
106 from = elt->buffer_index;
107 msg_type = elt->msg_type;
109 ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME);
110 ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
112 f = vlib_get_frame_to_node
113 (vm, 1 ? handoff_dispatch_node.index : ethernet_input_node.index);
115 to = vlib_frame_vector_args (f);
117 n_left_to_node = elt->n_vectors;
119 while (n_left_to_node >= 4)
130 while (n_left_to_node > 0)
138 vectors += elt->n_vectors;
139 f->n_vectors = elt->n_vectors;
140 vlib_put_frame_to_node
141 (vm, 1 ? handoff_dispatch_node.index : ethernet_input_node.index, f);
145 elt->msg_type = 0xfefefefe;
146 CLIB_MEMORY_BARRIER();
151 * Limit the number of packets pushed into the graph
153 if (vectors >= fq->vector_threshold)
155 fq->head_hint = fq->head;
163 int dpdk_frame_queue_dequeue (vlib_main_t *vm)
165 return vlib_frame_queue_dequeue_internal (vm);
169 * dpdk_worker_thread - Contains the main loop of a worker thread.
172 * Information for the current thread
174 * The name of thread performing dpdk device IO (if any). If there are no
175 * instances of that thread, then the current thread will do dpdk device
176 * polling. Ports will be divided among instances of the current thread.
178 * If not null, this function will be called once during each main loop.
180 static_always_inline void
181 dpdk_worker_thread_internal (vlib_main_t *vm,
182 dpdk_worker_thread_callback_t callback,
185 vlib_node_main_t * nm = &vm->node_main;
186 u64 cpu_time_now = clib_cpu_time_now ();
190 vlib_worker_thread_barrier_check ();
192 vlib_frame_queue_dequeue_internal (vm);
194 /* Invoke callback if supplied */
195 if (PREDICT_FALSE(callback != NULL))
198 if (!have_io_threads)
200 vlib_node_runtime_t * n;
201 vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
203 cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
204 VLIB_NODE_STATE_POLLING, /* frame */ 0,
210 if (_vec_len (nm->pending_frames))
213 cpu_time_now = clib_cpu_time_now ();
214 for (i = 0; i < _vec_len (nm->pending_frames); i++) {
215 vlib_pending_frame_t *p;
217 p = nm->pending_frames + i;
219 cpu_time_now = dispatch_pending_node (vm, p, cpu_time_now);
221 _vec_len (nm->pending_frames) = 0;
223 vlib_increment_main_loop_counter (vm);
225 /* Record time stamp in case there are no enabled nodes and above
226 calls do not update time stamp. */
227 cpu_time_now = clib_cpu_time_now ();
231 void dpdk_worker_thread (vlib_worker_thread_t * w,
233 dpdk_worker_thread_callback_t callback)
237 vlib_thread_main_t * tm = vlib_get_thread_main();
238 vlib_thread_registration_t * tr;
239 dpdk_main_t * dm = &dpdk_main;
241 vm = vlib_get_main();
243 ASSERT(vm->cpu_index == os_get_cpu_number());
245 clib_time_init (&vm->clib_time);
246 clib_mem_set_heap (w->thread_mheap);
248 /* Wait until the dpdk init sequence is complete */
249 while (dm->io_thread_release == 0)
250 vlib_worker_thread_barrier_check ();
252 /* any I/O threads? */
253 p = hash_get_mem (tm->thread_registrations_by_name, io_name);
254 tr = (vlib_thread_registration_t *)p[0];
256 if (tr && tr->count > 0)
257 dpdk_worker_thread_internal(vm, callback, /* have_io_threads */ 1);
259 dpdk_worker_thread_internal(vm, callback, /* have_io_threads */ 0);
262 void dpdk_worker_thread_fn (void * arg)
264 vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
265 vlib_worker_thread_init (w);
266 dpdk_worker_thread (w, "io", 0);
270 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
273 .function = dpdk_worker_thread_fn,
274 .mheap_size = 256<<20,
278 void dpdk_io_thread_fn (void * arg)
280 vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
281 vlib_worker_thread_init (w);
282 dpdk_io_thread (w, 0, 0, "workers", 0);
286 VLIB_REGISTER_THREAD (io_thread_reg, static) = {
289 .function = dpdk_io_thread_fn,
290 .mheap_size = 256<<20,
294 static clib_error_t *
295 dpdk_thread_init (vlib_main_t *vm)
300 VLIB_INIT_FUNCTION(dpdk_thread_init);