3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <vnet/vnet.h>
18 #include <vppinfra/xxhash.h>
19 #include <vlib/threads.h>
20 #include <vnet/handoff.h>
21 #include <vnet/feature/feature.h>
25 uword *workers_bitmap;
27 } per_inteface_handoff_data_t;
31 u32 cached_next_index;
33 u32 first_worker_index;
35 per_inteface_handoff_data_t *if_data;
37 /* Worker handoff index */
38 u32 frame_queue_index;
40 u64 (*hash_fn) (ethernet_header_t *);
43 handoff_main_t handoff_main;
48 u32 next_worker_index;
50 } worker_handoff_trace_t;
52 #define foreach_worker_handoff_error \
53 _(CONGESTION_DROP, "congestion drop")
57 #define _(sym,str) WORKER_HANDOFF_ERROR_##sym,
58 foreach_worker_handoff_error
60 WORKER_HANDOFF_N_ERROR,
61 } worker_handoff_error_t;
63 static char *worker_handoff_error_strings[] = {
64 #define _(sym,string) string,
65 foreach_worker_handoff_error
69 /* packet trace format function */
71 format_worker_handoff_trace (u8 * s, va_list * args)
73 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
74 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
75 worker_handoff_trace_t *t = va_arg (*args, worker_handoff_trace_t *);
78 format (s, "worker-handoff: sw_if_index %d, next_worker %d, buffer 0x%x",
79 t->sw_if_index, t->next_worker_index, t->buffer_index);
83 vlib_node_registration_t handoff_node;
86 worker_handoff_node_fn (vlib_main_t * vm,
87 vlib_node_runtime_t * node, vlib_frame_t * frame)
89 handoff_main_t *hm = &handoff_main;
90 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
91 u32 n_enq, n_left_from, *from;
92 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
94 from = vlib_frame_vector_args (frame);
95 n_left_from = frame->n_vectors;
96 vlib_get_buffers (vm, from, bufs, n_left_from);
101 while (n_left_from > 0)
106 per_inteface_handoff_data_t *ihd0;
110 sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
111 ASSERT (hm->if_data);
112 ihd0 = vec_elt_at_index (hm->if_data, sw_if_index0);
115 * Force unknown traffic onto worker 0,
116 * and into ethernet-input. $$$$ add more hashes.
119 /* Compute ingress LB hash */
120 hash_key = hm->hash_fn ((ethernet_header_t *)
121 vlib_buffer_get_current (b[0]));
122 hash = (u32) clib_xxhash (hash_key);
124 /* if input node did not specify next index, then packet
125 should go to eternet-input */
127 if (PREDICT_TRUE (is_pow2 (vec_len (ihd0->workers))))
128 index0 = hash & (vec_len (ihd0->workers) - 1);
130 index0 = hash % vec_len (ihd0->workers);
132 ti[0] = hm->first_worker_index + ihd0->workers[index0];
134 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
135 && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
137 worker_handoff_trace_t *t =
138 vlib_add_trace (vm, node, b[0], sizeof (*t));
139 t->sw_if_index = sw_if_index0;
140 t->next_worker_index = ti[0];
141 t->buffer_index = vlib_get_buffer_index (vm, b[0]);
150 n_enq = vlib_buffer_enqueue_to_thread (vm, hm->frame_queue_index, from,
151 thread_indices, frame->n_vectors, 1);
153 if (n_enq < frame->n_vectors)
154 vlib_node_increment_counter (vm, node->node_index,
155 WORKER_HANDOFF_ERROR_CONGESTION_DROP,
156 frame->n_vectors - n_enq);
157 return frame->n_vectors;
161 VLIB_REGISTER_NODE (worker_handoff_node) = {
162 .function = worker_handoff_node_fn,
163 .name = "worker-handoff",
164 .vector_size = sizeof (u32),
165 .format_trace = format_worker_handoff_trace,
166 .type = VLIB_NODE_TYPE_INTERNAL,
167 .n_errors = ARRAY_LEN(worker_handoff_error_strings),
168 .error_strings = worker_handoff_error_strings,
176 VLIB_NODE_FUNCTION_MULTIARCH (worker_handoff_node, worker_handoff_node_fn)
180 interface_handoff_enable_disable (vlib_main_t * vm, u32 sw_if_index,
181 uword * bitmap, int enable_disable)
183 handoff_main_t *hm = &handoff_main;
184 vnet_sw_interface_t *sw;
185 vnet_main_t *vnm = vnet_get_main ();
186 per_inteface_handoff_data_t *d;
189 if (pool_is_free_index (vnm->interface_main.sw_interfaces, sw_if_index))
190 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
192 sw = vnet_get_sw_interface (vnm, sw_if_index);
193 if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
194 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
196 if (clib_bitmap_last_set (bitmap) >= hm->num_workers)
197 return VNET_API_ERROR_INVALID_WORKER;
199 if (hm->frame_queue_index == ~0)
201 vlib_node_t *n = vlib_get_node_by_name (vm, (u8 *) "ethernet-input");
202 hm->frame_queue_index = vlib_frame_queue_main_init (n->index, 0);
205 vec_validate (hm->if_data, sw_if_index);
206 d = vec_elt_at_index (hm->if_data, sw_if_index);
208 vec_free (d->workers);
209 vec_free (d->workers_bitmap);
213 d->workers_bitmap = bitmap;
215 clib_bitmap_foreach (i, bitmap,
217 vec_add1(d->workers, i);
222 vnet_feature_enable_disable ("device-input", "worker-handoff",
223 sw_if_index, enable_disable, 0, 0);
227 static clib_error_t *
228 set_interface_handoff_command_fn (vlib_main_t * vm,
229 unformat_input_t * input,
230 vlib_cli_command_t * cmd)
232 handoff_main_t *hm = &handoff_main;
233 u32 sw_if_index = ~0;
234 int enable_disable = 1;
240 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
242 if (unformat (input, "disable"))
244 else if (unformat (input, "workers %U", unformat_bitmap_list, &bitmap))
246 else if (unformat (input, "%U", unformat_vnet_sw_interface,
247 vnet_get_main (), &sw_if_index))
249 else if (unformat (input, "symmetrical"))
251 else if (unformat (input, "asymmetrical"))
257 if (sw_if_index == ~0)
258 return clib_error_return (0, "Please specify an interface...");
261 return clib_error_return (0, "Please specify list of workers...");
264 interface_handoff_enable_disable (vm, sw_if_index, bitmap,
272 case VNET_API_ERROR_INVALID_SW_IF_INDEX:
273 return clib_error_return (0, "Invalid interface");
276 case VNET_API_ERROR_INVALID_WORKER:
277 return clib_error_return (0, "Invalid worker(s)");
280 case VNET_API_ERROR_UNIMPLEMENTED:
281 return clib_error_return (0,
282 "Device driver doesn't support redirection");
286 return clib_error_return (0, "unknown return value %d", rv);
290 hm->hash_fn = eth_get_sym_key;
292 hm->hash_fn = eth_get_key;
298 VLIB_CLI_COMMAND (set_interface_handoff_command, static) = {
299 .path = "set interface handoff",
301 "set interface handoff <interface-name> workers <workers-list> [symmetrical|asymmetrical]",
302 .function = set_interface_handoff_command_fn,
307 handoff_init (vlib_main_t * vm)
309 handoff_main_t *hm = &handoff_main;
310 vlib_thread_main_t *tm = vlib_get_thread_main ();
314 if ((error = vlib_call_init_function (vm, threads_init)))
317 vlib_thread_registration_t *tr;
318 /* Only the standard vnet worker threads are supported */
319 p = hash_get_mem (tm->thread_registrations_by_name, "workers");
322 tr = (vlib_thread_registration_t *) p[0];
325 hm->num_workers = tr->count;
326 hm->first_worker_index = tr->first_index;
330 hm->hash_fn = eth_get_key;
331 hm->frame_queue_index = ~0;
336 VLIB_INIT_FUNCTION (handoff_init);
339 * fd.io coding-style-patch-verification: ON
342 * eval: (c-set-style "gnu")