2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/ip/ip.h>
16 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
17 #include <vnet/classify/vnet_classify.h>
18 #include <vnet/dpo/classify_dpo.h>
24 } ip_classify_trace_t;
26 /* packet trace format function */
27 static u8 * format_ip_classify_trace (u8 * s, va_list * args)
29 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
30 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
31 ip_classify_trace_t * t = va_arg (*args, ip_classify_trace_t *);
33 s = format (s, "IP_CLASSIFY: next_index %d, table %d, entry %d",
34 t->next_index, t->table_index, t->entry_index);
38 vlib_node_registration_t ip4_classify_node;
39 vlib_node_registration_t ip6_classify_node;
41 #define foreach_ip_classify_error \
42 _(MISS, "Classify misses") \
43 _(HIT, "Classify hits") \
44 _(CHAIN_HIT, "Classify hits after chain walk")
47 #define _(sym,str) IP_CLASSIFY_ERROR_##sym,
48 foreach_ip_classify_error
51 } ip_classify_error_t;
53 static char * ip_classify_error_strings[] = {
54 #define _(sym,string) string,
55 foreach_ip_classify_error
60 ip_classify_inline (vlib_main_t * vm,
61 vlib_node_runtime_t * node,
62 vlib_frame_t * frame, int is_ip4)
64 u32 n_left_from, * from, * to_next;
65 ip_lookup_next_t next_index;
66 vnet_classify_main_t * vcm = &vnet_classify_main;
67 f64 now = vlib_time_now (vm);
74 n_next = IP4_LOOKUP_N_NEXT;
76 n_next = IP6_LOOKUP_N_NEXT;
79 from = vlib_frame_vector_args (frame);
80 n_left_from = frame->n_vectors;
82 /* First pass: compute hashes */
84 while (n_left_from > 2)
86 vlib_buffer_t * b0, * b1;
89 u32 cd_index0, cd_index1;
90 classify_dpo_t *cd0, * cd1;
91 u32 table_index0, table_index1;
92 vnet_classify_table_t * t0, * t1;
94 /* prefetch next iteration */
96 vlib_buffer_t * p1, * p2;
98 p1 = vlib_get_buffer (vm, from[1]);
99 p2 = vlib_get_buffer (vm, from[2]);
101 vlib_prefetch_buffer_header (p1, STORE);
102 CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
103 vlib_prefetch_buffer_header (p2, STORE);
104 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
108 b0 = vlib_get_buffer (vm, bi0);
109 h0 = (void *)vlib_buffer_get_current(b0) -
110 ethernet_buffer_header_size(b0);
113 b1 = vlib_get_buffer (vm, bi1);
114 h1 = (void *)vlib_buffer_get_current(b1) -
115 ethernet_buffer_header_size(b1);
117 cd_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
118 cd0 = classify_dpo_get(cd_index0);
119 table_index0 = cd0->cd_table_index;
121 cd_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
122 cd1 = classify_dpo_get(cd_index1);
123 table_index1 = cd1->cd_table_index;
125 t0 = pool_elt_at_index (vcm->tables, table_index0);
127 t1 = pool_elt_at_index (vcm->tables, table_index1);
129 vnet_buffer(b0)->l2_classify.hash =
130 vnet_classify_hash_packet (t0, (u8 *) h0);
132 vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
134 vnet_buffer(b1)->l2_classify.hash =
135 vnet_classify_hash_packet (t1, (u8 *) h1);
137 vnet_classify_prefetch_bucket (t1, vnet_buffer(b1)->l2_classify.hash);
139 vnet_buffer(b0)->l2_classify.table_index = table_index0;
141 vnet_buffer(b1)->l2_classify.table_index = table_index1;
147 while (n_left_from > 0)
155 vnet_classify_table_t * t0;
158 b0 = vlib_get_buffer (vm, bi0);
159 h0 = (void *)vlib_buffer_get_current(b0) -
160 ethernet_buffer_header_size(b0);
162 cd_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
163 cd0 = classify_dpo_get(cd_index0);
164 table_index0 = cd0->cd_table_index;
166 t0 = pool_elt_at_index (vcm->tables, table_index0);
167 vnet_buffer(b0)->l2_classify.hash =
168 vnet_classify_hash_packet (t0, (u8 *) h0);
170 vnet_buffer(b0)->l2_classify.table_index = table_index0;
171 vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
177 next_index = node->cached_next_index;
178 from = vlib_frame_vector_args (frame);
179 n_left_from = frame->n_vectors;
181 while (n_left_from > 0)
185 vlib_get_next_frame (vm, node, next_index,
186 to_next, n_left_to_next);
188 /* Not enough load/store slots to dual loop... */
189 while (n_left_from > 0 && n_left_to_next > 0)
193 u32 next0 = IP_LOOKUP_NEXT_DROP;
195 vnet_classify_table_t * t0;
196 vnet_classify_entry_t * e0;
200 /* Stride 3 seems to work best */
201 if (PREDICT_TRUE (n_left_from > 3))
203 vlib_buffer_t * p1 = vlib_get_buffer(vm, from[3]);
204 vnet_classify_table_t * tp1;
208 table_index1 = vnet_buffer(p1)->l2_classify.table_index;
210 if (PREDICT_TRUE (table_index1 != ~0))
212 tp1 = pool_elt_at_index (vcm->tables, table_index1);
213 phash1 = vnet_buffer(p1)->l2_classify.hash;
214 vnet_classify_prefetch_entry (tp1, phash1);
218 /* speculatively enqueue b0 to the current next frame */
226 b0 = vlib_get_buffer (vm, bi0);
228 table_index0 = vnet_buffer(b0)->l2_classify.table_index;
231 vnet_buffer(b0)->l2_classify.opaque_index = ~0;
233 if (PREDICT_TRUE(table_index0 != ~0))
235 hash0 = vnet_buffer(b0)->l2_classify.hash;
236 t0 = pool_elt_at_index (vcm->tables, table_index0);
238 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0,
242 vnet_buffer(b0)->l2_classify.opaque_index
244 vlib_buffer_advance (b0, e0->advance);
245 next0 = (e0->next_index < node->n_next_nodes)?
246 e0->next_index:next0;
253 if (t0->next_table_index != ~0)
254 t0 = pool_elt_at_index (vcm->tables,
255 t0->next_table_index);
258 next0 = (t0->miss_next_index < n_next) ?
259 t0->miss_next_index : next0;
264 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
265 e0 = vnet_classify_find_entry
266 (t0, (u8 *) h0, hash0, now);
269 vnet_buffer(b0)->l2_classify.opaque_index
271 vlib_buffer_advance (b0, e0->advance);
272 next0 = (e0->next_index < node->n_next_nodes)?
273 e0->next_index:next0;
282 if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
283 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
285 ip_classify_trace_t *t =
286 vlib_add_trace (vm, node, b0, sizeof (*t));
287 t->next_index = next0;
288 t->table_index = t0 ? t0 - vcm->tables : ~0;
289 t->entry_index = e0 ? e0 - t0->entries : ~0;
292 /* verify speculative enqueue, maybe switch current next frame */
293 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
294 to_next, n_left_to_next,
298 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
301 vlib_node_increment_counter (vm, node->node_index,
302 IP_CLASSIFY_ERROR_MISS,
304 vlib_node_increment_counter (vm, node->node_index,
305 IP_CLASSIFY_ERROR_HIT,
307 vlib_node_increment_counter (vm, node->node_index,
308 IP_CLASSIFY_ERROR_CHAIN_HIT,
310 return frame->n_vectors;
314 ip4_classify (vlib_main_t * vm,
315 vlib_node_runtime_t * node,
316 vlib_frame_t * frame)
318 return ip_classify_inline (vm, node, frame, 1 /* is_ip4 */);
322 VLIB_REGISTER_NODE (ip4_classify_node) = {
323 .function = ip4_classify,
324 .name = "ip4-classify",
325 .vector_size = sizeof (u32),
326 .sibling_of = "ip4-lookup",
327 .format_trace = format_ip_classify_trace,
328 .n_errors = ARRAY_LEN(ip_classify_error_strings),
329 .error_strings = ip_classify_error_strings,
334 VLIB_NODE_FUNCTION_MULTIARCH (ip4_classify_node, ip4_classify)
337 ip6_classify (vlib_main_t * vm,
338 vlib_node_runtime_t * node,
339 vlib_frame_t * frame)
341 return ip_classify_inline (vm, node, frame, 0 /* is_ip4 */);
345 VLIB_REGISTER_NODE (ip6_classify_node) = {
346 .function = ip6_classify,
347 .name = "ip6-classify",
348 .vector_size = sizeof (u32),
349 .sibling_of = "ip6-lookup",
350 .format_trace = format_ip_classify_trace,
351 .n_errors = ARRAY_LEN(ip_classify_error_strings),
352 .error_strings = ip_classify_error_strings,
357 VLIB_NODE_FUNCTION_MULTIARCH (ip6_classify_node, ip6_classify)
359 static clib_error_t *
360 ip_classify_init (vlib_main_t * vm)
365 VLIB_INIT_FUNCTION (ip_classify_init);