2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vlib/vlib.h>
19 #include <vnet/vnet.h>
20 #include <vnet/policer/policer.h>
21 #include <vnet/policer/police_inlines.h>
22 #include <vnet/ip/ip.h>
23 #include <vnet/classify/policer_classify.h>
24 #include <vnet/classify/vnet_classify.h>
25 #include <vnet/l2/feat_bitmap.h>
26 #include <vnet/l2/l2_input.h>
29 /* Dispatch functions meant to be instantiated elsewhere */
36 } vnet_policer_trace_t;
38 /* packet trace format function */
40 format_policer_trace (u8 * s, va_list * args)
42 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
43 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
44 vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
46 s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
47 t->sw_if_index, t->policer_index, t->next_index);
51 #define foreach_vnet_policer_error \
52 _(TRANSMIT, "Packets Transmitted") \
53 _(DROP, "Packets Dropped")
57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
58 foreach_vnet_policer_error
61 } vnet_policer_error_t;
63 static char *vnet_policer_error_strings[] = {
64 #define _(sym,string) string,
65 foreach_vnet_policer_error
70 vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
73 u32 n_left_from, *from, *to_next;
74 vnet_policer_next_t next_index;
75 vnet_policer_main_t *pm = &vnet_policer_main;
76 u64 time_in_policer_periods;
79 time_in_policer_periods =
80 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
82 from = vlib_frame_vector_args (frame);
83 n_left_from = frame->n_vectors;
84 next_index = node->cached_next_index;
86 while (n_left_from > 0)
90 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
92 while (n_left_from >= 4 && n_left_to_next >= 2)
95 vlib_buffer_t *b0, *b1;
97 u32 sw_if_index0, sw_if_index1;
101 /* Prefetch next iteration. */
103 vlib_buffer_t *b2, *b3;
105 b2 = vlib_get_buffer (vm, from[2]);
106 b3 = vlib_get_buffer (vm, from[3]);
108 vlib_prefetch_buffer_header (b2, LOAD);
109 vlib_prefetch_buffer_header (b3, LOAD);
112 /* speculatively enqueue b0 and b1 to the current next frame */
113 to_next[0] = bi0 = from[0];
114 to_next[1] = bi1 = from[1];
120 b0 = vlib_get_buffer (vm, bi0);
121 b1 = vlib_get_buffer (vm, bi1);
123 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
124 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
126 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
127 pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
129 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
130 POLICE_CONFORM /* no chaining */ );
132 act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
133 POLICE_CONFORM /* no chaining */ );
135 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
137 next0 = VNET_POLICER_NEXT_DROP;
138 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
140 else /* transmit or mark-and-transmit action */
143 vnet_feature_next (&next0, b0);
146 if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
148 next1 = VNET_POLICER_NEXT_DROP;
149 b1->error = node->errors[VNET_POLICER_ERROR_DROP];
151 else /* transmit or mark-and-transmit action */
154 vnet_feature_next (&next1, b1);
157 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
159 if (b0->flags & VLIB_BUFFER_IS_TRACED)
161 vnet_policer_trace_t *t =
162 vlib_add_trace (vm, node, b0, sizeof (*t));
163 t->sw_if_index = sw_if_index0;
164 t->next_index = next0;
166 if (b1->flags & VLIB_BUFFER_IS_TRACED)
168 vnet_policer_trace_t *t =
169 vlib_add_trace (vm, node, b1, sizeof (*t));
170 t->sw_if_index = sw_if_index1;
171 t->next_index = next1;
175 /* verify speculative enqueues, maybe switch current next frame */
176 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
177 to_next, n_left_to_next,
178 bi0, bi1, next0, next1);
181 while (n_left_from > 0 && n_left_to_next > 0)
197 b0 = vlib_get_buffer (vm, bi0);
199 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
201 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
203 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
204 POLICE_CONFORM /* no chaining */ );
206 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
208 next0 = VNET_POLICER_NEXT_DROP;
209 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
211 else /* transmit or mark-and-transmit action */
214 vnet_feature_next (&next0, b0);
217 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
218 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
220 vnet_policer_trace_t *t =
221 vlib_add_trace (vm, node, b0, sizeof (*t));
222 t->sw_if_index = sw_if_index0;
223 t->next_index = next0;
224 t->policer_index = pi0;
227 /* verify speculative enqueue, maybe switch current next frame */
228 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
229 to_next, n_left_to_next,
233 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
236 vlib_node_increment_counter (vm, node->node_index,
237 VNET_POLICER_ERROR_TRANSMIT, transmitted);
238 return frame->n_vectors;
241 VLIB_NODE_FN (policer_input_node)
242 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
244 return vnet_policer_inline (vm, node, frame);
247 VLIB_REGISTER_NODE (policer_input_node) = {
248 .name = "policer-input",
249 .vector_size = sizeof (u32),
250 .format_trace = format_policer_trace,
251 .type = VLIB_NODE_TYPE_INTERNAL,
252 .n_errors = ARRAY_LEN(vnet_policer_error_strings),
253 .error_strings = vnet_policer_error_strings,
254 .n_next_nodes = VNET_POLICER_N_NEXT,
256 [VNET_POLICER_NEXT_DROP] = "error-drop",
260 VNET_FEATURE_INIT (policer_input_node, static) = {
261 .arc_name = "device-input",
262 .node_name = "policer-input",
263 .runs_before = VNET_FEATURES ("ethernet-input"),
273 } policer_classify_trace_t;
276 format_policer_classify_trace (u8 * s, va_list * args)
278 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
279 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
280 policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
282 s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
284 t->sw_if_index, t->next_index, t->table_index, t->offset,
289 #define foreach_policer_classify_error \
290 _(MISS, "Policer classify misses") \
291 _(HIT, "Policer classify hits") \
292 _(CHAIN_HIT, "Policer classify hits after chain walk") \
293 _(DROP, "Policer classify action drop")
297 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
298 foreach_policer_classify_error
300 POLICER_CLASSIFY_N_ERROR,
301 } policer_classify_error_t;
303 static char *policer_classify_error_strings[] = {
304 #define _(sym,string) string,
305 foreach_policer_classify_error
310 policer_classify_inline (vlib_main_t * vm,
311 vlib_node_runtime_t * node,
312 vlib_frame_t * frame,
313 policer_classify_table_id_t tid)
315 u32 n_left_from, *from, *to_next;
316 policer_classify_next_index_t next_index;
317 policer_classify_main_t *pcm = &policer_classify_main;
318 vnet_classify_main_t *vcm = pcm->vnet_classify_main;
319 f64 now = vlib_time_now (vm);
324 u64 time_in_policer_periods;
326 time_in_policer_periods =
327 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
329 n_next_nodes = node->n_next_nodes;
331 from = vlib_frame_vector_args (frame);
332 n_left_from = frame->n_vectors;
334 /* First pass: compute hashes */
335 while (n_left_from > 2)
337 vlib_buffer_t *b0, *b1;
340 u32 sw_if_index0, sw_if_index1;
341 u32 table_index0, table_index1;
342 vnet_classify_table_t *t0, *t1;
344 /* Prefetch next iteration */
346 vlib_buffer_t *p1, *p2;
348 p1 = vlib_get_buffer (vm, from[1]);
349 p2 = vlib_get_buffer (vm, from[2]);
351 vlib_prefetch_buffer_header (p1, STORE);
352 CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
353 vlib_prefetch_buffer_header (p2, STORE);
354 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
358 b0 = vlib_get_buffer (vm, bi0);
362 b1 = vlib_get_buffer (vm, bi1);
365 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
367 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
369 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
371 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
373 t0 = pool_elt_at_index (vcm->tables, table_index0);
375 t1 = pool_elt_at_index (vcm->tables, table_index1);
377 vnet_buffer (b0)->l2_classify.hash =
378 vnet_classify_hash_packet (t0, (u8 *) h0);
380 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
382 vnet_buffer (b1)->l2_classify.hash =
383 vnet_classify_hash_packet (t1, (u8 *) h1);
385 vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
387 vnet_buffer (b0)->l2_classify.table_index = table_index0;
389 vnet_buffer (b1)->l2_classify.table_index = table_index1;
395 while (n_left_from > 0)
402 vnet_classify_table_t *t0;
405 b0 = vlib_get_buffer (vm, bi0);
408 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
410 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
412 t0 = pool_elt_at_index (vcm->tables, table_index0);
413 vnet_buffer (b0)->l2_classify.hash =
414 vnet_classify_hash_packet (t0, (u8 *) h0);
416 vnet_buffer (b0)->l2_classify.table_index = table_index0;
417 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
423 next_index = node->cached_next_index;
424 from = vlib_frame_vector_args (frame);
425 n_left_from = frame->n_vectors;
427 while (n_left_from > 0)
431 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
433 /* Not enough load/store slots to dual loop... */
434 while (n_left_from > 0 && n_left_to_next > 0)
438 u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
440 vnet_classify_table_t *t0;
441 vnet_classify_entry_t *e0;
446 /* Stride 3 seems to work best */
447 if (PREDICT_TRUE (n_left_from > 3))
449 vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
450 vnet_classify_table_t *tp1;
454 table_index1 = vnet_buffer (p1)->l2_classify.table_index;
456 if (PREDICT_TRUE (table_index1 != ~0))
458 tp1 = pool_elt_at_index (vcm->tables, table_index1);
459 phash1 = vnet_buffer (p1)->l2_classify.hash;
460 vnet_classify_prefetch_entry (tp1, phash1);
464 /* Speculatively enqueue b0 to the current next frame */
472 b0 = vlib_get_buffer (vm, bi0);
474 table_index0 = vnet_buffer (b0)->l2_classify.table_index;
478 if (tid == POLICER_CLASSIFY_TABLE_L2)
480 /* Feature bitmap update and determine the next node */
481 next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
482 L2INPUT_FEAT_POLICER_CLAS);
485 vnet_get_config_data (pcm->vnet_config_main[tid],
486 &b0->current_config_index, &next0,
487 /* # bytes of config data */ 0);
489 vnet_buffer (b0)->l2_classify.opaque_index = ~0;
491 if (PREDICT_TRUE (table_index0 != ~0))
493 hash0 = vnet_buffer (b0)->l2_classify.hash;
494 t0 = pool_elt_at_index (vcm->tables, table_index0);
495 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
499 act0 = vnet_policer_police (vm,
502 time_in_policer_periods,
504 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
506 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
507 b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
515 if (PREDICT_TRUE (t0->next_table_index != ~0))
517 t0 = pool_elt_at_index (vcm->tables,
518 t0->next_table_index);
522 next0 = (t0->miss_next_index < n_next_nodes) ?
523 t0->miss_next_index : next0;
528 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
530 vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
533 act0 = vnet_policer_police (vm,
536 time_in_policer_periods,
538 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
540 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
542 node->errors[POLICER_CLASSIFY_ERROR_DROP];
551 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
552 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
554 policer_classify_trace_t *t =
555 vlib_add_trace (vm, node, b0, sizeof (*t));
556 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
557 t->next_index = next0;
558 t->table_index = t0 ? t0 - vcm->tables : ~0;
559 t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
560 t->policer_index = e0 ? e0->next_index : ~0;
563 /* Verify speculative enqueue, maybe switch current next frame */
564 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
565 n_left_to_next, bi0, next0);
568 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
571 vlib_node_increment_counter (vm, node->node_index,
572 POLICER_CLASSIFY_ERROR_MISS, misses);
573 vlib_node_increment_counter (vm, node->node_index,
574 POLICER_CLASSIFY_ERROR_HIT, hits);
575 vlib_node_increment_counter (vm, node->node_index,
576 POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
578 return frame->n_vectors;
581 VLIB_NODE_FN (ip4_policer_classify_node) (vlib_main_t * vm,
582 vlib_node_runtime_t * node,
583 vlib_frame_t * frame)
585 return policer_classify_inline (vm, node, frame,
586 POLICER_CLASSIFY_TABLE_IP4);
590 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
591 .name = "ip4-policer-classify",
592 .vector_size = sizeof (u32),
593 .format_trace = format_policer_classify_trace,
594 .n_errors = ARRAY_LEN(policer_classify_error_strings),
595 .error_strings = policer_classify_error_strings,
596 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
598 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
603 VLIB_NODE_FN (ip6_policer_classify_node) (vlib_main_t * vm,
604 vlib_node_runtime_t * node,
605 vlib_frame_t * frame)
607 return policer_classify_inline (vm, node, frame,
608 POLICER_CLASSIFY_TABLE_IP6);
612 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
613 .name = "ip6-policer-classify",
614 .vector_size = sizeof (u32),
615 .format_trace = format_policer_classify_trace,
616 .n_errors = ARRAY_LEN(policer_classify_error_strings),
617 .error_strings = policer_classify_error_strings,
618 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
620 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
625 VLIB_NODE_FN (l2_policer_classify_node) (vlib_main_t * vm,
626 vlib_node_runtime_t * node,
627 vlib_frame_t * frame)
629 return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
633 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
634 .name = "l2-policer-classify",
635 .vector_size = sizeof (u32),
636 .format_trace = format_policer_classify_trace,
637 .n_errors = ARRAY_LEN (policer_classify_error_strings),
638 .error_strings = policer_classify_error_strings,
639 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
641 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
646 #ifndef CLIB_MARCH_VARIANT
647 static clib_error_t *
648 policer_classify_init (vlib_main_t * vm)
650 policer_classify_main_t *pcm = &policer_classify_main;
653 pcm->vnet_main = vnet_get_main ();
654 pcm->vnet_classify_main = &vnet_classify_main;
656 /* Initialize L2 feature next-node indexes */
657 feat_bitmap_init_next_nodes (vm,
658 l2_policer_classify_node.index,
660 l2input_get_feat_names (),
661 pcm->feat_next_node_index);
666 VLIB_INIT_FUNCTION (policer_classify_init);
667 #endif /* CLIB_MARCH_VARIANT */
670 * fd.io coding-style-patch-verification: ON
673 * eval: (c-set-style "gnu")