2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vlib/vlib.h>
19 #include <vnet/vnet.h>
20 #include <vnet/policer/policer.h>
21 #include <vnet/policer/police_inlines.h>
22 #include <vnet/ip/ip.h>
23 #include <vnet/classify/policer_classify.h>
24 #include <vnet/classify/vnet_classify.h>
25 #include <vnet/l2/feat_bitmap.h>
26 #include <vnet/l2/l2_input.h>
29 /* Dispatch functions meant to be instantiated elsewhere */
36 } vnet_policer_trace_t;
38 /* packet trace format function */
40 format_policer_trace (u8 * s, va_list * args)
42 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
43 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
44 vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
46 s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
47 t->sw_if_index, t->policer_index, t->next_index);
51 #define foreach_vnet_policer_error \
52 _(TRANSMIT, "Packets Transmitted") \
53 _(DROP, "Packets Dropped")
57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
58 foreach_vnet_policer_error
61 } vnet_policer_error_t;
63 static char *vnet_policer_error_strings[] = {
64 #define _(sym,string) string,
65 foreach_vnet_policer_error
70 vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
73 u32 n_left_from, *from, *to_next;
74 vnet_policer_next_t next_index;
75 vnet_policer_main_t *pm = &vnet_policer_main;
76 u64 time_in_policer_periods;
79 time_in_policer_periods =
80 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
82 from = vlib_frame_vector_args (frame);
83 n_left_from = frame->n_vectors;
84 next_index = node->cached_next_index;
86 while (n_left_from > 0)
90 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
92 while (n_left_from >= 4 && n_left_to_next >= 2)
95 vlib_buffer_t *b0, *b1;
97 u32 sw_if_index0, sw_if_index1;
101 /* Prefetch next iteration. */
103 vlib_buffer_t *b2, *b3;
105 b2 = vlib_get_buffer (vm, from[2]);
106 b3 = vlib_get_buffer (vm, from[3]);
108 vlib_prefetch_buffer_header (b2, LOAD);
109 vlib_prefetch_buffer_header (b3, LOAD);
112 /* speculatively enqueue b0 and b1 to the current next frame */
113 to_next[0] = bi0 = from[0];
114 to_next[1] = bi1 = from[1];
120 b0 = vlib_get_buffer (vm, bi0);
121 b1 = vlib_get_buffer (vm, bi1);
123 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
124 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
126 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
127 pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
129 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
130 POLICE_CONFORM /* no chaining */, true);
132 act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
133 POLICE_CONFORM /* no chaining */, true);
135 if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
137 next0 = VNET_POLICER_NEXT_HANDOFF;
138 vnet_buffer (b0)->policer.index = pi0;
140 else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
142 next0 = VNET_POLICER_NEXT_DROP;
143 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
145 else /* transmit or mark-and-transmit action */
148 vnet_feature_next (&next0, b0);
151 if (PREDICT_FALSE (act1 == QOS_ACTION_HANDOFF))
153 next1 = VNET_POLICER_NEXT_HANDOFF;
154 vnet_buffer (b1)->policer.index = pi1;
156 else if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
158 next1 = VNET_POLICER_NEXT_DROP;
159 b1->error = node->errors[VNET_POLICER_ERROR_DROP];
161 else /* transmit or mark-and-transmit action */
164 vnet_feature_next (&next1, b1);
167 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
169 if (b0->flags & VLIB_BUFFER_IS_TRACED)
171 vnet_policer_trace_t *t =
172 vlib_add_trace (vm, node, b0, sizeof (*t));
173 t->sw_if_index = sw_if_index0;
174 t->next_index = next0;
176 if (b1->flags & VLIB_BUFFER_IS_TRACED)
178 vnet_policer_trace_t *t =
179 vlib_add_trace (vm, node, b1, sizeof (*t));
180 t->sw_if_index = sw_if_index1;
181 t->next_index = next1;
185 /* verify speculative enqueues, maybe switch current next frame */
186 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
187 to_next, n_left_to_next,
188 bi0, bi1, next0, next1);
191 while (n_left_from > 0 && n_left_to_next > 0)
207 b0 = vlib_get_buffer (vm, bi0);
209 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
211 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
213 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
214 POLICE_CONFORM /* no chaining */, true);
216 if (PREDICT_FALSE (act0 == QOS_ACTION_HANDOFF))
218 next0 = VNET_POLICER_NEXT_HANDOFF;
219 vnet_buffer (b0)->policer.index = pi0;
221 else if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
223 next0 = VNET_POLICER_NEXT_DROP;
224 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
226 else /* transmit or mark-and-transmit action */
229 vnet_feature_next (&next0, b0);
232 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
233 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
235 vnet_policer_trace_t *t =
236 vlib_add_trace (vm, node, b0, sizeof (*t));
237 t->sw_if_index = sw_if_index0;
238 t->next_index = next0;
239 t->policer_index = pi0;
242 /* verify speculative enqueue, maybe switch current next frame */
243 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
244 to_next, n_left_to_next,
248 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
251 vlib_node_increment_counter (vm, node->node_index,
252 VNET_POLICER_ERROR_TRANSMIT, transmitted);
253 return frame->n_vectors;
256 VLIB_NODE_FN (policer_input_node)
257 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
259 return vnet_policer_inline (vm, node, frame);
262 VLIB_REGISTER_NODE (policer_input_node) = {
263 .name = "policer-input",
264 .vector_size = sizeof (u32),
265 .format_trace = format_policer_trace,
266 .type = VLIB_NODE_TYPE_INTERNAL,
267 .n_errors = ARRAY_LEN(vnet_policer_error_strings),
268 .error_strings = vnet_policer_error_strings,
269 .n_next_nodes = VNET_POLICER_N_NEXT,
271 [VNET_POLICER_NEXT_DROP] = "error-drop",
272 [VNET_POLICER_NEXT_HANDOFF] = "policer-input-handoff",
276 VNET_FEATURE_INIT (policer_input_node, static) = {
277 .arc_name = "device-input",
278 .node_name = "policer-input",
279 .runs_before = VNET_FEATURES ("ethernet-input"),
282 static char *policer_input_handoff_error_strings[] = { "congestion drop" };
284 VLIB_NODE_FN (policer_input_handoff_node)
285 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
287 return policer_handoff (vm, node, frame, vnet_policer_main.fq_index, ~0);
290 VLIB_REGISTER_NODE (policer_input_handoff_node) = {
291 .name = "policer-input-handoff",
292 .vector_size = sizeof (u32),
293 .format_trace = format_policer_handoff_trace,
294 .type = VLIB_NODE_TYPE_INTERNAL,
295 .n_errors = ARRAY_LEN(policer_input_handoff_error_strings),
296 .error_strings = policer_input_handoff_error_strings,
311 } policer_classify_trace_t;
314 format_policer_classify_trace (u8 * s, va_list * args)
316 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
317 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
318 policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
320 s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
322 t->sw_if_index, t->next_index, t->table_index, t->offset,
327 #define foreach_policer_classify_error \
328 _(MISS, "Policer classify misses") \
329 _(HIT, "Policer classify hits") \
330 _(CHAIN_HIT, "Policer classify hits after chain walk") \
331 _(DROP, "Policer classify action drop")
335 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
336 foreach_policer_classify_error
338 POLICER_CLASSIFY_N_ERROR,
339 } policer_classify_error_t;
341 static char *policer_classify_error_strings[] = {
342 #define _(sym,string) string,
343 foreach_policer_classify_error
348 policer_classify_inline (vlib_main_t * vm,
349 vlib_node_runtime_t * node,
350 vlib_frame_t * frame,
351 policer_classify_table_id_t tid)
353 u32 n_left_from, *from, *to_next;
354 policer_classify_next_index_t next_index;
355 policer_classify_main_t *pcm = &policer_classify_main;
356 vnet_classify_main_t *vcm = pcm->vnet_classify_main;
357 f64 now = vlib_time_now (vm);
362 u64 time_in_policer_periods;
364 time_in_policer_periods =
365 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
367 n_next_nodes = node->n_next_nodes;
369 from = vlib_frame_vector_args (frame);
370 n_left_from = frame->n_vectors;
372 /* First pass: compute hashes */
373 while (n_left_from > 2)
375 vlib_buffer_t *b0, *b1;
378 u32 sw_if_index0, sw_if_index1;
379 u32 table_index0, table_index1;
380 vnet_classify_table_t *t0, *t1;
382 /* Prefetch next iteration */
384 vlib_buffer_t *p1, *p2;
386 p1 = vlib_get_buffer (vm, from[1]);
387 p2 = vlib_get_buffer (vm, from[2]);
389 vlib_prefetch_buffer_header (p1, STORE);
390 CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
391 vlib_prefetch_buffer_header (p2, STORE);
392 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
396 b0 = vlib_get_buffer (vm, bi0);
400 b1 = vlib_get_buffer (vm, bi1);
403 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
405 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
407 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
409 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
411 t0 = pool_elt_at_index (vcm->tables, table_index0);
413 t1 = pool_elt_at_index (vcm->tables, table_index1);
415 vnet_buffer (b0)->l2_classify.hash =
416 vnet_classify_hash_packet (t0, (u8 *) h0);
418 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
420 vnet_buffer (b1)->l2_classify.hash =
421 vnet_classify_hash_packet (t1, (u8 *) h1);
423 vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
425 vnet_buffer (b0)->l2_classify.table_index = table_index0;
427 vnet_buffer (b1)->l2_classify.table_index = table_index1;
433 while (n_left_from > 0)
440 vnet_classify_table_t *t0;
443 b0 = vlib_get_buffer (vm, bi0);
446 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
448 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
450 t0 = pool_elt_at_index (vcm->tables, table_index0);
451 vnet_buffer (b0)->l2_classify.hash =
452 vnet_classify_hash_packet (t0, (u8 *) h0);
454 vnet_buffer (b0)->l2_classify.table_index = table_index0;
455 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
461 next_index = node->cached_next_index;
462 from = vlib_frame_vector_args (frame);
463 n_left_from = frame->n_vectors;
465 while (n_left_from > 0)
469 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
471 /* Not enough load/store slots to dual loop... */
472 while (n_left_from > 0 && n_left_to_next > 0)
476 u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
478 vnet_classify_table_t *t0;
479 vnet_classify_entry_t *e0;
484 /* Stride 3 seems to work best */
485 if (PREDICT_TRUE (n_left_from > 3))
487 vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
488 vnet_classify_table_t *tp1;
492 table_index1 = vnet_buffer (p1)->l2_classify.table_index;
494 if (PREDICT_TRUE (table_index1 != ~0))
496 tp1 = pool_elt_at_index (vcm->tables, table_index1);
497 phash1 = vnet_buffer (p1)->l2_classify.hash;
498 vnet_classify_prefetch_entry (tp1, phash1);
502 /* Speculatively enqueue b0 to the current next frame */
510 b0 = vlib_get_buffer (vm, bi0);
512 table_index0 = vnet_buffer (b0)->l2_classify.table_index;
516 if (tid == POLICER_CLASSIFY_TABLE_L2)
518 /* Feature bitmap update and determine the next node */
519 next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
520 L2INPUT_FEAT_POLICER_CLAS);
523 vnet_get_config_data (pcm->vnet_config_main[tid],
524 &b0->current_config_index, &next0,
525 /* # bytes of config data */ 0);
527 vnet_buffer (b0)->l2_classify.opaque_index = ~0;
529 if (PREDICT_TRUE (table_index0 != ~0))
531 hash0 = vnet_buffer (b0)->l2_classify.hash;
532 t0 = pool_elt_at_index (vcm->tables, table_index0);
533 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
537 act0 = vnet_policer_police (vm, b0, e0->next_index,
538 time_in_policer_periods,
539 e0->opaque_index, false);
540 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
542 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
543 b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
551 if (PREDICT_TRUE (t0->next_table_index != ~0))
553 t0 = pool_elt_at_index (vcm->tables,
554 t0->next_table_index);
558 next0 = (t0->miss_next_index < n_next_nodes) ?
559 t0->miss_next_index : next0;
564 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
566 vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
569 act0 = vnet_policer_police (vm, b0, e0->next_index,
570 time_in_policer_periods,
571 e0->opaque_index, false);
572 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
574 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
576 node->errors[POLICER_CLASSIFY_ERROR_DROP];
585 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
586 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
588 policer_classify_trace_t *t =
589 vlib_add_trace (vm, node, b0, sizeof (*t));
590 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
591 t->next_index = next0;
592 t->table_index = t0 ? t0 - vcm->tables : ~0;
593 t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
594 t->policer_index = e0 ? e0->next_index : ~0;
597 /* Verify speculative enqueue, maybe switch current next frame */
598 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
599 n_left_to_next, bi0, next0);
602 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
605 vlib_node_increment_counter (vm, node->node_index,
606 POLICER_CLASSIFY_ERROR_MISS, misses);
607 vlib_node_increment_counter (vm, node->node_index,
608 POLICER_CLASSIFY_ERROR_HIT, hits);
609 vlib_node_increment_counter (vm, node->node_index,
610 POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
612 return frame->n_vectors;
615 VLIB_NODE_FN (ip4_policer_classify_node) (vlib_main_t * vm,
616 vlib_node_runtime_t * node,
617 vlib_frame_t * frame)
619 return policer_classify_inline (vm, node, frame,
620 POLICER_CLASSIFY_TABLE_IP4);
624 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
625 .name = "ip4-policer-classify",
626 .vector_size = sizeof (u32),
627 .format_trace = format_policer_classify_trace,
628 .n_errors = ARRAY_LEN(policer_classify_error_strings),
629 .error_strings = policer_classify_error_strings,
630 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
632 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
637 VLIB_NODE_FN (ip6_policer_classify_node) (vlib_main_t * vm,
638 vlib_node_runtime_t * node,
639 vlib_frame_t * frame)
641 return policer_classify_inline (vm, node, frame,
642 POLICER_CLASSIFY_TABLE_IP6);
646 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
647 .name = "ip6-policer-classify",
648 .vector_size = sizeof (u32),
649 .format_trace = format_policer_classify_trace,
650 .n_errors = ARRAY_LEN(policer_classify_error_strings),
651 .error_strings = policer_classify_error_strings,
652 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
654 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
659 VLIB_NODE_FN (l2_policer_classify_node) (vlib_main_t * vm,
660 vlib_node_runtime_t * node,
661 vlib_frame_t * frame)
663 return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
667 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
668 .name = "l2-policer-classify",
669 .vector_size = sizeof (u32),
670 .format_trace = format_policer_classify_trace,
671 .n_errors = ARRAY_LEN (policer_classify_error_strings),
672 .error_strings = policer_classify_error_strings,
673 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
675 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
680 #ifndef CLIB_MARCH_VARIANT
681 static clib_error_t *
682 policer_classify_init (vlib_main_t * vm)
684 policer_classify_main_t *pcm = &policer_classify_main;
687 pcm->vnet_main = vnet_get_main ();
688 pcm->vnet_classify_main = &vnet_classify_main;
690 /* Initialize L2 feature next-node indexes */
691 feat_bitmap_init_next_nodes (vm,
692 l2_policer_classify_node.index,
694 l2input_get_feat_names (),
695 pcm->feat_next_node_index);
700 VLIB_INIT_FUNCTION (policer_classify_init);
701 #endif /* CLIB_MARCH_VARIANT */
704 * fd.io coding-style-patch-verification: ON
707 * eval: (c-set-style "gnu")