3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <vnet/vnet.h>
18 #include <vnet/hash/hash.h>
19 #include <vlib/threads.h>
20 #include <vnet/feature/feature.h>
24 vnet_hash_fn_t hash_fn;
25 uword *workers_bitmap;
27 } per_inteface_handoff_data_t;
31 u32 cached_next_index;
33 u32 first_worker_index;
35 per_inteface_handoff_data_t *if_data;
37 /* Worker handoff index */
38 u32 frame_queue_index;
41 extern handoff_main_t handoff_main;
43 #ifndef CLIB_MARCH_VARIANT
45 handoff_main_t handoff_main;
47 #endif /* CLIB_MARCH_VARIANT */
52 u32 next_worker_index;
54 } worker_handoff_trace_t;
56 #define foreach_worker_handoff_error \
57 _(CONGESTION_DROP, "congestion drop")
61 #define _(sym,str) WORKER_HANDOFF_ERROR_##sym,
62 foreach_worker_handoff_error
64 WORKER_HANDOFF_N_ERROR,
65 } worker_handoff_error_t;
67 static char *worker_handoff_error_strings[] = {
68 #define _(sym,string) string,
69 foreach_worker_handoff_error
73 /* packet trace format function */
75 format_worker_handoff_trace (u8 * s, va_list * args)
77 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
78 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
79 worker_handoff_trace_t *t = va_arg (*args, worker_handoff_trace_t *);
81 s = format (s, "worker-handoff: sw_if_index %d, next_worker %d, buffer 0x%x",
82 t->sw_if_index, t->next_worker_index, t->buffer_index);
87 worker_handoff_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node,
88 vlib_buffer_t **bufs, u16 *threads, u32 n_vectors)
90 worker_handoff_trace_t *t;
99 t = vlib_add_trace (vm, node, b[0], sizeof (*t));
100 t->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
101 t->next_worker_index = ti[0];
102 t->buffer_index = vlib_get_buffer_index (vm, b[0]);
110 VLIB_NODE_FN (worker_handoff_node) (vlib_main_t * vm,
111 vlib_node_runtime_t * node,
112 vlib_frame_t * frame)
114 handoff_main_t *hm = &handoff_main;
115 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
116 u32 n_enq, n_left_from, *from;
117 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
119 from = vlib_frame_vector_args (frame);
120 n_left_from = frame->n_vectors;
121 vlib_get_buffers (vm, from, bufs, n_left_from);
126 while (n_left_from > 0)
128 per_inteface_handoff_data_t *ihd0;
129 u32 sw_if_index0, hash, index0;
132 sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
133 ihd0 = vec_elt_at_index (hm->if_data, sw_if_index0);
135 /* Compute ingress LB hash */
136 data = vlib_buffer_get_current (b[0]);
137 ihd0->hash_fn (&data, &hash, 1);
139 /* if input node did not specify next index, then packet
140 should go to ethernet-input */
142 if (PREDICT_TRUE (is_pow2 (vec_len (ihd0->workers))))
143 index0 = hash & (vec_len (ihd0->workers) - 1);
145 index0 = hash % vec_len (ihd0->workers);
147 ti[0] = hm->first_worker_index + ihd0->workers[index0];
155 if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
156 worker_handoff_trace_frame (vm, node, bufs, thread_indices,
159 n_enq = vlib_buffer_enqueue_to_thread (vm, node, hm->frame_queue_index, from,
160 thread_indices, frame->n_vectors, 1);
162 if (n_enq < frame->n_vectors)
163 vlib_node_increment_counter (vm, node->node_index,
164 WORKER_HANDOFF_ERROR_CONGESTION_DROP,
165 frame->n_vectors - n_enq);
166 return frame->n_vectors;
169 VLIB_REGISTER_NODE (worker_handoff_node) = {
170 .name = "worker-handoff",
171 .vector_size = sizeof (u32),
172 .format_trace = format_worker_handoff_trace,
173 .type = VLIB_NODE_TYPE_INTERNAL,
174 .n_errors = ARRAY_LEN(worker_handoff_error_strings),
175 .error_strings = worker_handoff_error_strings,
183 #ifndef CLIB_MARCH_VARIANT
186 interface_handoff_enable_disable (vlib_main_t *vm, u32 sw_if_index,
187 uword *bitmap, u8 is_sym, int is_l4,
190 handoff_main_t *hm = &handoff_main;
191 vnet_sw_interface_t *sw;
192 vnet_main_t *vnm = vnet_get_main ();
193 per_inteface_handoff_data_t *d;
196 if (pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index))
197 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
199 sw = vnet_get_sw_interface (vnm, sw_if_index);
200 if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
201 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
203 if (clib_bitmap_last_set (bitmap) >= hm->num_workers)
204 return VNET_API_ERROR_INVALID_WORKER;
206 if (hm->frame_queue_index == ~0)
208 vlib_node_t *n = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
209 hm->frame_queue_index = vlib_frame_queue_main_init (n->index, 0);
212 vec_validate (hm->if_data, sw_if_index);
213 d = vec_elt_at_index (hm->if_data, sw_if_index);
215 vec_free (d->workers);
216 vec_free (d->workers_bitmap);
220 d->workers_bitmap = bitmap;
221 clib_bitmap_foreach (i, bitmap)
223 vec_add1(d->workers, i);
229 return VNET_API_ERROR_UNIMPLEMENTED;
231 d->hash_fn = vnet_hash_function_from_name (
232 "handoff-eth-sym", VNET_HASH_FN_TYPE_ETHERNET);
238 vnet_hash_default_function (VNET_HASH_FN_TYPE_ETHERNET);
240 d->hash_fn = vnet_hash_function_from_name (
241 "handoff-eth", VNET_HASH_FN_TYPE_ETHERNET);
245 vnet_feature_enable_disable ("device-input", "worker-handoff",
246 sw_if_index, enable_disable, 0, 0);
250 static clib_error_t *
251 set_interface_handoff_command_fn (vlib_main_t * vm,
252 unformat_input_t * input,
253 vlib_cli_command_t * cmd)
255 u32 sw_if_index = ~0, is_sym = 0, is_l4 = 0;
256 int enable_disable = 1;
260 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
262 if (unformat (input, "disable"))
264 else if (unformat (input, "workers %U", unformat_bitmap_list, &bitmap))
266 else if (unformat (input, "%U", unformat_vnet_sw_interface,
267 vnet_get_main (), &sw_if_index))
269 else if (unformat (input, "symmetrical"))
271 else if (unformat (input, "asymmetrical"))
273 else if (unformat (input, "l4"))
279 if (sw_if_index == ~0)
280 return clib_error_return (0, "Please specify an interface...");
283 return clib_error_return (0, "Please specify list of workers...");
285 rv = interface_handoff_enable_disable (vm, sw_if_index, bitmap, is_sym,
286 is_l4, enable_disable);
293 case VNET_API_ERROR_INVALID_SW_IF_INDEX:
294 return clib_error_return (0, "Invalid interface");
297 case VNET_API_ERROR_INVALID_WORKER:
298 return clib_error_return (0, "Invalid worker(s)");
301 case VNET_API_ERROR_UNIMPLEMENTED:
302 return clib_error_return (0,
303 "Device driver doesn't support redirection");
307 return clib_error_return (0, "unknown return value %d", rv);
314 VLIB_CLI_COMMAND (set_interface_handoff_command, static) = {
315 .path = "set interface handoff",
316 .short_help = "set interface handoff <interface-name> workers <workers-list>"
317 " [symmetrical|asymmetrical]",
318 .function = set_interface_handoff_command_fn,
323 handoff_init (vlib_main_t * vm)
325 handoff_main_t *hm = &handoff_main;
326 vlib_thread_main_t *tm = vlib_get_thread_main ();
330 if ((error = vlib_call_init_function (vm, threads_init)))
333 vlib_thread_registration_t *tr;
334 /* Only the standard vnet worker threads are supported */
335 p = hash_get_mem (tm->thread_registrations_by_name, "workers");
338 tr = (vlib_thread_registration_t *) p[0];
341 hm->num_workers = tr->count;
342 hm->first_worker_index = tr->first_index;
346 hm->frame_queue_index = ~0;
351 VLIB_INIT_FUNCTION (handoff_init);
353 #endif /* CLIB_MARCH_VARIANT */
355 * fd.io coding-style-patch-verification: ON
358 * eval: (c-set-style "gnu")