2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/ip/ip.h>
16 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
17 #include <vnet/classify/vnet_classify.h>
23 } ip_classify_trace_t;
25 /* packet trace format function */
26 static u8 * format_ip_classify_trace (u8 * s, va_list * args)
28 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
29 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
30 ip_classify_trace_t * t = va_arg (*args, ip_classify_trace_t *);
32 s = format (s, "IP_CLASSIFY: next_index %d, table %d, entry %d",
33 t->next_index, t->table_index, t->entry_index);
37 vlib_node_registration_t ip4_classify_node;
38 vlib_node_registration_t ip6_classify_node;
40 #define foreach_ip_classify_error \
41 _(MISS, "Classify misses") \
42 _(HIT, "Classify hits") \
43 _(CHAIN_HIT, "Classify hits after chain walk")
46 #define _(sym,str) IP_CLASSIFY_ERROR_##sym,
47 foreach_ip_classify_error
50 } ip_classify_error_t;
52 static char * ip_classify_error_strings[] = {
53 #define _(sym,string) string,
54 foreach_ip_classify_error
59 ip_classify_inline (vlib_main_t * vm,
60 vlib_node_runtime_t * node,
61 vlib_frame_t * frame, int is_ip4)
63 u32 n_left_from, * from, * to_next;
64 ip_lookup_next_t next_index;
65 vnet_classify_main_t * vcm = &vnet_classify_main;
66 ip_lookup_main_t * lm;
67 f64 now = vlib_time_now (vm);
73 lm = &ip4_main.lookup_main;
75 lm = &ip6_main.lookup_main;
77 from = vlib_frame_vector_args (frame);
78 n_left_from = frame->n_vectors;
80 /* First pass: compute hashes */
82 while (n_left_from > 2)
84 vlib_buffer_t * b0, * b1;
87 u32 adj_index0, adj_index1;
88 ip_adjacency_t * adj0, * adj1;
89 u32 table_index0, table_index1;
90 vnet_classify_table_t * t0, * t1;
92 /* prefetch next iteration */
94 vlib_buffer_t * p1, * p2;
96 p1 = vlib_get_buffer (vm, from[1]);
97 p2 = vlib_get_buffer (vm, from[2]);
99 vlib_prefetch_buffer_header (p1, STORE);
100 CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
101 vlib_prefetch_buffer_header (p2, STORE);
102 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
106 b0 = vlib_get_buffer (vm, bi0);
110 b1 = vlib_get_buffer (vm, bi1);
113 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
114 adj0 = ip_get_adjacency (lm, adj_index0);
115 table_index0 = adj0->classify_table_index;
117 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
118 adj1 = ip_get_adjacency (lm, adj_index1);
119 table_index1 = adj1->classify_table_index;
121 t0 = pool_elt_at_index (vcm->tables, table_index0);
123 t1 = pool_elt_at_index (vcm->tables, table_index1);
125 vnet_buffer(b0)->l2_classify.hash =
126 vnet_classify_hash_packet (t0, (u8 *) h0);
128 vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
130 vnet_buffer(b1)->l2_classify.hash =
131 vnet_classify_hash_packet (t1, (u8 *) h1);
133 vnet_classify_prefetch_bucket (t1, vnet_buffer(b1)->l2_classify.hash);
135 vnet_buffer(b0)->l2_classify.table_index = table_index0;
137 vnet_buffer(b1)->l2_classify.table_index = table_index1;
143 while (n_left_from > 0)
149 ip_adjacency_t * adj0;
151 vnet_classify_table_t * t0;
154 b0 = vlib_get_buffer (vm, bi0);
157 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
158 adj0 = ip_get_adjacency (lm, adj_index0);
159 table_index0 = adj0->classify_table_index;
161 t0 = pool_elt_at_index (vcm->tables, table_index0);
162 vnet_buffer(b0)->l2_classify.hash =
163 vnet_classify_hash_packet (t0, (u8 *) h0);
165 vnet_buffer(b0)->l2_classify.table_index = table_index0;
166 vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
172 next_index = node->cached_next_index;
173 from = vlib_frame_vector_args (frame);
174 n_left_from = frame->n_vectors;
176 while (n_left_from > 0)
180 vlib_get_next_frame (vm, node, next_index,
181 to_next, n_left_to_next);
183 /* Not enough load/store slots to dual loop... */
184 while (n_left_from > 0 && n_left_to_next > 0)
188 u32 next0 = IP_LOOKUP_NEXT_MISS;
190 vnet_classify_table_t * t0;
191 vnet_classify_entry_t * e0;
195 /* Stride 3 seems to work best */
196 if (PREDICT_TRUE (n_left_from > 3))
198 vlib_buffer_t * p1 = vlib_get_buffer(vm, from[3]);
199 vnet_classify_table_t * tp1;
203 table_index1 = vnet_buffer(p1)->l2_classify.table_index;
205 if (PREDICT_TRUE (table_index1 != ~0))
207 tp1 = pool_elt_at_index (vcm->tables, table_index1);
208 phash1 = vnet_buffer(p1)->l2_classify.hash;
209 vnet_classify_prefetch_entry (tp1, phash1);
213 /* speculatively enqueue b0 to the current next frame */
221 b0 = vlib_get_buffer (vm, bi0);
223 table_index0 = vnet_buffer(b0)->l2_classify.table_index;
226 vnet_buffer(b0)->l2_classify.opaque_index = ~0;
228 if (PREDICT_TRUE(table_index0 != ~0))
230 hash0 = vnet_buffer(b0)->l2_classify.hash;
231 t0 = pool_elt_at_index (vcm->tables, table_index0);
233 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0,
237 vnet_buffer(b0)->l2_classify.opaque_index
239 vlib_buffer_advance (b0, e0->advance);
240 next0 = (e0->next_index < IP_LOOKUP_N_NEXT)?
241 e0->next_index:next0;
248 if (t0->next_table_index != ~0)
249 t0 = pool_elt_at_index (vcm->tables,
250 t0->next_table_index);
253 next0 = (t0->miss_next_index < IP_LOOKUP_N_NEXT)?
254 t0->miss_next_index:next0;
259 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
260 e0 = vnet_classify_find_entry
261 (t0, (u8 *) h0, hash0, now);
264 vnet_buffer(b0)->l2_classify.opaque_index
266 vlib_buffer_advance (b0, e0->advance);
267 next0 = (e0->next_index < IP_LOOKUP_N_NEXT)?
268 e0->next_index:next0;
277 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
278 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
280 ip_classify_trace_t *t =
281 vlib_add_trace (vm, node, b0, sizeof (*t));
282 t->next_index = next0;
283 t->table_index = t0 ? t0 - vcm->tables : ~0;
284 t->entry_index = e0 ? e0 - t0->entries : ~0;
287 /* verify speculative enqueue, maybe switch current next frame */
288 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
289 to_next, n_left_to_next,
293 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
296 vlib_node_increment_counter (vm, node->node_index,
297 IP_CLASSIFY_ERROR_MISS,
299 vlib_node_increment_counter (vm, node->node_index,
300 IP_CLASSIFY_ERROR_HIT,
302 vlib_node_increment_counter (vm, node->node_index,
303 IP_CLASSIFY_ERROR_CHAIN_HIT,
305 return frame->n_vectors;
309 ip4_classify (vlib_main_t * vm,
310 vlib_node_runtime_t * node,
311 vlib_frame_t * frame)
313 return ip_classify_inline (vm, node, frame, 1 /* is_ip4 */);
317 VLIB_REGISTER_NODE (ip4_classify_node) = {
318 .function = ip4_classify,
319 .name = "ip4-classify",
320 .vector_size = sizeof (u32),
321 .format_trace = format_ip_classify_trace,
322 .n_errors = ARRAY_LEN(ip_classify_error_strings),
323 .error_strings = ip_classify_error_strings,
325 .n_next_nodes = IP_LOOKUP_N_NEXT,
327 [IP_LOOKUP_NEXT_MISS] = "ip4-miss",
328 [IP_LOOKUP_NEXT_DROP] = "ip4-drop",
329 [IP_LOOKUP_NEXT_PUNT] = "ip4-punt",
330 [IP_LOOKUP_NEXT_LOCAL] = "ip4-local",
331 [IP_LOOKUP_NEXT_ARP] = "ip4-arp",
332 [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite-transit",
333 [IP_LOOKUP_NEXT_CLASSIFY] = "ip4-classify", /* probably not... */
334 [IP_LOOKUP_NEXT_MAP] = "ip4-map",
335 [IP_LOOKUP_NEXT_MAP_T] = "ip4-map-t",
336 [IP_LOOKUP_NEXT_SIXRD] = "ip4-sixrd",
337 [IP_LOOKUP_NEXT_HOP_BY_HOP] = "ip4-hop-by-hop",
338 [IP_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip4-add-hop-by-hop",
339 [IP_LOOKUP_NEXT_POP_HOP_BY_HOP] = "ip4-pop-hop-by-hop",
344 ip6_classify (vlib_main_t * vm,
345 vlib_node_runtime_t * node,
346 vlib_frame_t * frame)
348 return ip_classify_inline (vm, node, frame, 0 /* is_ip4 */);
352 VLIB_REGISTER_NODE (ip6_classify_node) = {
353 .function = ip6_classify,
354 .name = "ip6-classify",
355 .vector_size = sizeof (u32),
356 .format_trace = format_ip_classify_trace,
357 .n_errors = ARRAY_LEN(ip_classify_error_strings),
358 .error_strings = ip_classify_error_strings,
360 .n_next_nodes = IP_LOOKUP_N_NEXT,
362 [IP_LOOKUP_NEXT_MISS] = "ip6-miss",
363 [IP_LOOKUP_NEXT_DROP] = "ip6-drop",
364 [IP_LOOKUP_NEXT_PUNT] = "ip6-punt",
365 [IP_LOOKUP_NEXT_LOCAL] = "ip6-local",
366 [IP_LOOKUP_NEXT_ARP] = "ip6-discover-neighbor",
367 [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite",
368 [IP_LOOKUP_NEXT_CLASSIFY] = "ip6-classify", /* probably not... */
369 [IP_LOOKUP_NEXT_MAP] = "ip6-map",
370 [IP_LOOKUP_NEXT_MAP_T] = "ip6-map-t",
371 [IP_LOOKUP_NEXT_SIXRD] = "ip6-sixrd",
372 [IP_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop",
373 [IP_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop",
374 [IP_LOOKUP_NEXT_POP_HOP_BY_HOP] = "ip6-pop-hop-by-hop",
378 static clib_error_t *
379 ip_classify_init (vlib_main_t * vm)
384 VLIB_INIT_FUNCTION (ip_classify_init);