2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vlib/vlib.h>
19 #include <vnet/vnet.h>
20 #include <vnet/policer/policer.h>
21 #include <vnet/policer/police_inlines.h>
22 #include <vnet/ip/ip.h>
23 #include <vnet/classify/policer_classify.h>
24 #include <vnet/classify/vnet_classify.h>
25 #include <vnet/l2/feat_bitmap.h>
26 #include <vnet/l2/l2_input.h>
29 /* Dispatch functions meant to be instantiated elsewhere */
36 } vnet_policer_trace_t;
38 /* packet trace format function */
40 format_policer_trace (u8 * s, va_list * args)
42 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
43 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
44 vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
46 s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
47 t->sw_if_index, t->policer_index, t->next_index);
51 #define foreach_vnet_policer_error \
52 _(TRANSMIT, "Packets Transmitted") \
53 _(DROP, "Packets Dropped")
57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
58 foreach_vnet_policer_error
61 } vnet_policer_error_t;
63 static char *vnet_policer_error_strings[] = {
64 #define _(sym,string) string,
65 foreach_vnet_policer_error
70 vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
71 vlib_frame_t *frame, vlib_dir_t dir)
73 u32 n_left_from, *from, *to_next;
74 vnet_policer_next_t next_index;
75 vnet_policer_main_t *pm = &vnet_policer_main;
76 u64 time_in_policer_periods;
79 time_in_policer_periods =
80 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
82 from = vlib_frame_vector_args (frame);
83 n_left_from = frame->n_vectors;
84 next_index = node->cached_next_index;
86 while (n_left_from > 0)
90 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
92 while (n_left_from >= 4 && n_left_to_next >= 2)
95 vlib_buffer_t *b0, *b1;
97 u32 sw_if_index0, sw_if_index1;
101 /* Prefetch next iteration. */
103 vlib_buffer_t *b2, *b3;
105 b2 = vlib_get_buffer (vm, from[2]);
106 b3 = vlib_get_buffer (vm, from[3]);
108 vlib_prefetch_buffer_header (b2, LOAD);
109 vlib_prefetch_buffer_header (b3, LOAD);
112 /* speculatively enqueue b0 and b1 to the current next frame */
113 to_next[0] = bi0 = from[0];
114 to_next[1] = bi1 = from[1];
120 b0 = vlib_get_buffer (vm, bi0);
121 b1 = vlib_get_buffer (vm, bi1);
123 sw_if_index0 = vnet_buffer (b0)->sw_if_index[dir];
124 sw_if_index1 = vnet_buffer (b1)->sw_if_index[dir];
126 pi0 = pm->policer_index_by_sw_if_index[dir][sw_if_index0];
127 pi1 = pm->policer_index_by_sw_if_index[dir][sw_if_index1];
129 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
130 POLICE_CONFORM /* no chaining */, true);
132 act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
133 POLICE_CONFORM /* no chaining */, true);
135 if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
137 next0 = VNET_POLICER_NEXT_HANDOFF;
138 vnet_buffer (b0)->policer.index = pi0;
140 else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
142 next0 = VNET_POLICER_NEXT_DROP;
143 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
145 else /* transmit or mark-and-transmit action */
148 vnet_feature_next (&next0, b0);
151 if (PREDICT_FALSE (act1 == QOS_ACTION_HANDOFF))
153 next1 = VNET_POLICER_NEXT_HANDOFF;
154 vnet_buffer (b1)->policer.index = pi1;
156 else if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
158 next1 = VNET_POLICER_NEXT_DROP;
159 b1->error = node->errors[VNET_POLICER_ERROR_DROP];
161 else /* transmit or mark-and-transmit action */
164 vnet_feature_next (&next1, b1);
167 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
169 if (b0->flags & VLIB_BUFFER_IS_TRACED)
171 vnet_policer_trace_t *t =
172 vlib_add_trace (vm, node, b0, sizeof (*t));
173 t->sw_if_index = sw_if_index0;
174 t->next_index = next0;
176 if (b1->flags & VLIB_BUFFER_IS_TRACED)
178 vnet_policer_trace_t *t =
179 vlib_add_trace (vm, node, b1, sizeof (*t));
180 t->sw_if_index = sw_if_index1;
181 t->next_index = next1;
185 /* verify speculative enqueues, maybe switch current next frame */
186 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
187 to_next, n_left_to_next,
188 bi0, bi1, next0, next1);
191 while (n_left_from > 0 && n_left_to_next > 0)
207 b0 = vlib_get_buffer (vm, bi0);
209 sw_if_index0 = vnet_buffer (b0)->sw_if_index[dir];
210 pi0 = pm->policer_index_by_sw_if_index[dir][sw_if_index0];
212 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
213 POLICE_CONFORM /* no chaining */, true);
215 if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
217 next0 = VNET_POLICER_NEXT_HANDOFF;
218 vnet_buffer (b0)->policer.index = pi0;
220 else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
222 next0 = VNET_POLICER_NEXT_DROP;
223 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
225 else /* transmit or mark-and-transmit action */
228 vnet_feature_next (&next0, b0);
231 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
232 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
234 vnet_policer_trace_t *t =
235 vlib_add_trace (vm, node, b0, sizeof (*t));
236 t->sw_if_index = sw_if_index0;
237 t->next_index = next0;
238 t->policer_index = pi0;
241 /* verify speculative enqueue, maybe switch current next frame */
242 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
243 to_next, n_left_to_next,
247 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
250 vlib_node_increment_counter (vm, node->node_index,
251 VNET_POLICER_ERROR_TRANSMIT, transmitted);
252 return frame->n_vectors;
255 VLIB_NODE_FN (policer_input_node)
256 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
258 return vnet_policer_inline (vm, node, frame, VLIB_RX);
261 VLIB_REGISTER_NODE (policer_input_node) = {
262 .name = "policer-input",
263 .vector_size = sizeof (u32),
264 .format_trace = format_policer_trace,
265 .type = VLIB_NODE_TYPE_INTERNAL,
266 .n_errors = ARRAY_LEN(vnet_policer_error_strings),
267 .error_strings = vnet_policer_error_strings,
268 .n_next_nodes = VNET_POLICER_N_NEXT,
270 [VNET_POLICER_NEXT_DROP] = "error-drop",
271 [VNET_POLICER_NEXT_HANDOFF] = "policer-input-handoff",
275 VNET_FEATURE_INIT (policer_input_node, static) = {
276 .arc_name = "device-input",
277 .node_name = "policer-input",
278 .runs_before = VNET_FEATURES ("ethernet-input"),
281 VLIB_NODE_FN (policer_output_node)
282 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
284 return vnet_policer_inline (vm, node, frame, VLIB_TX);
287 VLIB_REGISTER_NODE (policer_output_node) = {
288 .name = "policer-output",
289 .vector_size = sizeof (u32),
290 .format_trace = format_policer_trace,
291 .type = VLIB_NODE_TYPE_INTERNAL,
292 .n_errors = ARRAY_LEN(vnet_policer_error_strings),
293 .error_strings = vnet_policer_error_strings,
294 .n_next_nodes = VNET_POLICER_N_NEXT,
296 [VNET_POLICER_NEXT_DROP] = "error-drop",
297 [VNET_POLICER_NEXT_HANDOFF] = "policer-output-handoff",
301 VNET_FEATURE_INIT (policer_output_node, static) = {
302 .arc_name = "ip4-output",
303 .node_name = "policer-output",
306 VNET_FEATURE_INIT (policer6_output_node, static) = {
307 .arc_name = "ip6-output",
308 .node_name = "policer-output",
311 static char *policer_input_handoff_error_strings[] = { "congestion drop" };
313 VLIB_NODE_FN (policer_input_handoff_node)
314 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
316 return policer_handoff (vm, node, frame, vnet_policer_main.fq_index[VLIB_RX],
320 VLIB_REGISTER_NODE (policer_input_handoff_node) = {
321 .name = "policer-input-handoff",
322 .vector_size = sizeof (u32),
323 .format_trace = format_policer_handoff_trace,
324 .type = VLIB_NODE_TYPE_INTERNAL,
325 .n_errors = ARRAY_LEN(policer_input_handoff_error_strings),
326 .error_strings = policer_input_handoff_error_strings,
334 VLIB_NODE_FN (policer_output_handoff_node)
335 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
337 return policer_handoff (vm, node, frame, vnet_policer_main.fq_index[VLIB_TX],
341 VLIB_REGISTER_NODE (policer_output_handoff_node) = {
342 .name = "policer-output-handoff",
343 .vector_size = sizeof (u32),
344 .format_trace = format_policer_handoff_trace,
345 .type = VLIB_NODE_TYPE_INTERNAL,
346 .n_errors = ARRAY_LEN(policer_input_handoff_error_strings),
347 .error_strings = policer_input_handoff_error_strings,
361 } policer_classify_trace_t;
364 format_policer_classify_trace (u8 * s, va_list * args)
366 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
367 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
368 policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
370 s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
372 t->sw_if_index, t->next_index, t->table_index, t->offset,
377 #define foreach_policer_classify_error \
378 _(MISS, "Policer classify misses") \
379 _(HIT, "Policer classify hits") \
380 _(CHAIN_HIT, "Policer classify hits after chain walk") \
381 _(DROP, "Policer classify action drop")
385 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
386 foreach_policer_classify_error
388 POLICER_CLASSIFY_N_ERROR,
389 } policer_classify_error_t;
391 static char *policer_classify_error_strings[] = {
392 #define _(sym,string) string,
393 foreach_policer_classify_error
398 policer_classify_inline (vlib_main_t * vm,
399 vlib_node_runtime_t * node,
400 vlib_frame_t * frame,
401 policer_classify_table_id_t tid)
403 u32 n_left_from, *from, *to_next;
404 policer_classify_next_index_t next_index;
405 policer_classify_main_t *pcm = &policer_classify_main;
406 vnet_classify_main_t *vcm = pcm->vnet_classify_main;
407 f64 now = vlib_time_now (vm);
412 u64 time_in_policer_periods;
414 time_in_policer_periods =
415 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
417 n_next_nodes = node->n_next_nodes;
419 from = vlib_frame_vector_args (frame);
420 n_left_from = frame->n_vectors;
422 /* First pass: compute hashes */
423 while (n_left_from > 2)
425 vlib_buffer_t *b0, *b1;
428 u32 sw_if_index0, sw_if_index1;
429 u32 table_index0, table_index1;
430 vnet_classify_table_t *t0, *t1;
432 /* Prefetch next iteration */
434 vlib_buffer_t *p1, *p2;
436 p1 = vlib_get_buffer (vm, from[1]);
437 p2 = vlib_get_buffer (vm, from[2]);
439 vlib_prefetch_buffer_header (p1, STORE);
440 clib_prefetch_store (p1->data);
441 vlib_prefetch_buffer_header (p2, STORE);
442 clib_prefetch_store (p2->data);
446 b0 = vlib_get_buffer (vm, bi0);
450 b1 = vlib_get_buffer (vm, bi1);
453 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
455 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
457 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
459 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
461 t0 = pool_elt_at_index (vcm->tables, table_index0);
463 t1 = pool_elt_at_index (vcm->tables, table_index1);
465 vnet_buffer (b0)->l2_classify.hash =
466 vnet_classify_hash_packet (t0, (u8 *) h0);
468 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
470 vnet_buffer (b1)->l2_classify.hash =
471 vnet_classify_hash_packet (t1, (u8 *) h1);
473 vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
475 vnet_buffer (b0)->l2_classify.table_index = table_index0;
477 vnet_buffer (b1)->l2_classify.table_index = table_index1;
483 while (n_left_from > 0)
490 vnet_classify_table_t *t0;
493 b0 = vlib_get_buffer (vm, bi0);
496 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
498 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
500 t0 = pool_elt_at_index (vcm->tables, table_index0);
501 vnet_buffer (b0)->l2_classify.hash =
502 vnet_classify_hash_packet (t0, (u8 *) h0);
504 vnet_buffer (b0)->l2_classify.table_index = table_index0;
505 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
511 next_index = node->cached_next_index;
512 from = vlib_frame_vector_args (frame);
513 n_left_from = frame->n_vectors;
515 while (n_left_from > 0)
519 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
521 /* Not enough load/store slots to dual loop... */
522 while (n_left_from > 0 && n_left_to_next > 0)
526 u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
528 vnet_classify_table_t *t0;
529 vnet_classify_entry_t *e0;
534 /* Stride 3 seems to work best */
535 if (PREDICT_TRUE (n_left_from > 3))
537 vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
538 vnet_classify_table_t *tp1;
542 table_index1 = vnet_buffer (p1)->l2_classify.table_index;
544 if (PREDICT_TRUE (table_index1 != ~0))
546 tp1 = pool_elt_at_index (vcm->tables, table_index1);
547 phash1 = vnet_buffer (p1)->l2_classify.hash;
548 vnet_classify_prefetch_entry (tp1, phash1);
552 /* Speculatively enqueue b0 to the current next frame */
560 b0 = vlib_get_buffer (vm, bi0);
562 table_index0 = vnet_buffer (b0)->l2_classify.table_index;
566 if (tid == POLICER_CLASSIFY_TABLE_L2)
568 /* Feature bitmap update and determine the next node */
569 next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
570 L2INPUT_FEAT_POLICER_CLAS);
573 vnet_get_config_data (pcm->vnet_config_main[tid],
574 &b0->current_config_index, &next0,
575 /* # bytes of config data */ 0);
577 vnet_buffer (b0)->l2_classify.opaque_index = ~0;
579 if (PREDICT_TRUE (table_index0 != ~0))
581 hash0 = vnet_buffer (b0)->l2_classify.hash;
582 t0 = pool_elt_at_index (vcm->tables, table_index0);
583 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
587 act0 = vnet_policer_police (vm, b0, e0->next_index,
588 time_in_policer_periods,
589 e0->opaque_index, false);
590 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
592 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
593 b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
601 if (PREDICT_TRUE (t0->next_table_index != ~0))
603 t0 = pool_elt_at_index (vcm->tables,
604 t0->next_table_index);
608 next0 = (t0->miss_next_index < n_next_nodes) ?
609 t0->miss_next_index : next0;
614 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
616 vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
619 act0 = vnet_policer_police (vm, b0, e0->next_index,
620 time_in_policer_periods,
621 e0->opaque_index, false);
622 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
624 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
626 node->errors[POLICER_CLASSIFY_ERROR_DROP];
635 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
636 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
638 policer_classify_trace_t *t =
639 vlib_add_trace (vm, node, b0, sizeof (*t));
640 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
641 t->next_index = next0;
642 t->table_index = t0 ? t0 - vcm->tables : ~0;
643 t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
644 t->policer_index = e0 ? e0->next_index : ~0;
647 /* Verify speculative enqueue, maybe switch current next frame */
648 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
649 n_left_to_next, bi0, next0);
652 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
655 vlib_node_increment_counter (vm, node->node_index,
656 POLICER_CLASSIFY_ERROR_MISS, misses);
657 vlib_node_increment_counter (vm, node->node_index,
658 POLICER_CLASSIFY_ERROR_HIT, hits);
659 vlib_node_increment_counter (vm, node->node_index,
660 POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
662 return frame->n_vectors;
665 VLIB_NODE_FN (ip4_policer_classify_node) (vlib_main_t * vm,
666 vlib_node_runtime_t * node,
667 vlib_frame_t * frame)
669 return policer_classify_inline (vm, node, frame,
670 POLICER_CLASSIFY_TABLE_IP4);
674 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
675 .name = "ip4-policer-classify",
676 .vector_size = sizeof (u32),
677 .format_trace = format_policer_classify_trace,
678 .n_errors = ARRAY_LEN(policer_classify_error_strings),
679 .error_strings = policer_classify_error_strings,
680 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
682 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
687 VLIB_NODE_FN (ip6_policer_classify_node) (vlib_main_t * vm,
688 vlib_node_runtime_t * node,
689 vlib_frame_t * frame)
691 return policer_classify_inline (vm, node, frame,
692 POLICER_CLASSIFY_TABLE_IP6);
696 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
697 .name = "ip6-policer-classify",
698 .vector_size = sizeof (u32),
699 .format_trace = format_policer_classify_trace,
700 .n_errors = ARRAY_LEN(policer_classify_error_strings),
701 .error_strings = policer_classify_error_strings,
702 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
704 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
709 VLIB_NODE_FN (l2_policer_classify_node) (vlib_main_t * vm,
710 vlib_node_runtime_t * node,
711 vlib_frame_t * frame)
713 return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
717 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
718 .name = "l2-policer-classify",
719 .vector_size = sizeof (u32),
720 .format_trace = format_policer_classify_trace,
721 .n_errors = ARRAY_LEN (policer_classify_error_strings),
722 .error_strings = policer_classify_error_strings,
723 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
725 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
730 #ifndef CLIB_MARCH_VARIANT
731 static clib_error_t *
732 policer_classify_init (vlib_main_t * vm)
734 policer_classify_main_t *pcm = &policer_classify_main;
737 pcm->vnet_main = vnet_get_main ();
738 pcm->vnet_classify_main = &vnet_classify_main;
740 /* Initialize L2 feature next-node indexes */
741 feat_bitmap_init_next_nodes (vm,
742 l2_policer_classify_node.index,
744 l2input_get_feat_names (),
745 pcm->feat_next_node_index);
750 VLIB_INIT_FUNCTION (policer_classify_init);
751 #endif /* CLIB_MARCH_VARIANT */
754 * fd.io coding-style-patch-verification: ON
757 * eval: (c-set-style "gnu")