2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/classify/vnet_classify.h>
16 #include <vnet/classify/input_acl.h>
17 #include <vnet/ip/ip.h>
18 #include <vnet/api_errno.h> /* for API error numbers */
19 #include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
20 #include <vnet/fib/fib_table.h>
22 vnet_classify_main_t vnet_classify_main;
24 #if VALIDATION_SCAFFOLDING
25 /* Validation scaffolding */
26 void mv (vnet_classify_table_t * t)
30 oldheap = clib_mem_set_heap (t->mheap);
32 clib_mem_set_heap (oldheap);
35 void rogue (vnet_classify_table_t * t)
38 vnet_classify_entry_t * v, * save_v;
39 u32 active_elements = 0;
40 vnet_classify_bucket_t * b;
42 for (i = 0; i < t->nbuckets; i++)
47 save_v = vnet_classify_get_entry (t, b->offset);
48 for (j = 0; j < (1<<b->log2_pages); j++)
50 for (k = 0; k < t->entries_per_page; k++)
52 v = vnet_classify_entry_at_index
53 (t, save_v, j*t->entries_per_page + k);
55 if (vnet_classify_entry_is_busy (v))
61 if (active_elements != t->active_elements)
62 clib_warning ("found %u expected %u elts", active_elements,
66 void mv (vnet_classify_table_t * t) { }
67 void rogue (vnet_classify_table_t * t) { }
70 void vnet_classify_register_unformat_l2_next_index_fn (unformat_function_t * fn)
72 vnet_classify_main_t * cm = &vnet_classify_main;
74 vec_add1 (cm->unformat_l2_next_index_fns, fn);
77 void vnet_classify_register_unformat_ip_next_index_fn (unformat_function_t * fn)
79 vnet_classify_main_t * cm = &vnet_classify_main;
81 vec_add1 (cm->unformat_ip_next_index_fns, fn);
85 vnet_classify_register_unformat_acl_next_index_fn (unformat_function_t * fn)
87 vnet_classify_main_t * cm = &vnet_classify_main;
89 vec_add1 (cm->unformat_acl_next_index_fns, fn);
93 vnet_classify_register_unformat_policer_next_index_fn (unformat_function_t * fn)
95 vnet_classify_main_t * cm = &vnet_classify_main;
97 vec_add1 (cm->unformat_policer_next_index_fns, fn);
100 void vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * fn)
102 vnet_classify_main_t * cm = &vnet_classify_main;
104 vec_add1 (cm->unformat_opaque_index_fns, fn);
107 vnet_classify_table_t *
108 vnet_classify_new_table (vnet_classify_main_t *cm,
109 u8 * mask, u32 nbuckets, u32 memory_size,
113 vnet_classify_table_t * t;
116 nbuckets = 1 << (max_log2 (nbuckets));
118 pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
119 memset(t, 0, sizeof (*t));
121 vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof(u32x4));
122 clib_memcpy (t->mask, mask, match_n_vectors * sizeof (u32x4));
124 t->next_table_index = ~0;
125 t->nbuckets = nbuckets;
126 t->log2_nbuckets = max_log2 (nbuckets);
127 t->match_n_vectors = match_n_vectors;
128 t->skip_n_vectors = skip_n_vectors;
129 t->entries_per_page = 2;
131 t->mheap = mheap_alloc (0 /* use VM */, memory_size);
133 vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
134 oldheap = clib_mem_set_heap (t->mheap);
136 t->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
137 CLIB_CACHE_LINE_BYTES);
138 t->writer_lock[0] = 0;
140 clib_mem_set_heap (oldheap);
144 void vnet_classify_delete_table_index (vnet_classify_main_t *cm,
145 u32 table_index, int del_chain)
147 vnet_classify_table_t * t;
149 /* Tolerate multiple frees, up to a point */
150 if (pool_is_free_index (cm->tables, table_index))
153 t = pool_elt_at_index (cm->tables, table_index);
154 if (del_chain && t->next_table_index != ~0)
155 /* Recursively delete the entire chain */
156 vnet_classify_delete_table_index (cm, t->next_table_index, del_chain);
159 vec_free (t->buckets);
160 mheap_free (t->mheap);
162 pool_put (cm->tables, t);
165 static vnet_classify_entry_t *
166 vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
168 vnet_classify_entry_t * rv = 0;
172 ASSERT (t->writer_lock[0]);
174 (sizeof(vnet_classify_entry_t) + (t->match_n_vectors*sizeof(u32x4)))
175 * t->entries_per_page * (1<<log2_pages);
177 if (log2_pages >= vec_len (t->freelists) || t->freelists [log2_pages] == 0)
179 oldheap = clib_mem_set_heap (t->mheap);
181 vec_validate (t->freelists, log2_pages);
183 rv = clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
184 clib_mem_set_heap (oldheap);
187 rv = t->freelists[log2_pages];
188 t->freelists[log2_pages] = rv->next_free;
193 memset (rv, 0xff, required_length);
198 vnet_classify_entry_free (vnet_classify_table_t * t,
199 vnet_classify_entry_t * v, u32 log2_pages)
201 ASSERT (t->writer_lock[0]);
203 ASSERT(vec_len (t->freelists) > log2_pages);
205 v->next_free = t->freelists[log2_pages];
206 t->freelists[log2_pages] = v;
209 static inline void make_working_copy
210 (vnet_classify_table_t * t, vnet_classify_bucket_t * b)
212 vnet_classify_entry_t * v;
213 vnet_classify_bucket_t working_bucket __attribute__((aligned (8)));
215 vnet_classify_entry_t * working_copy;
216 u32 thread_index = vlib_get_thread_index();
217 int working_copy_length, required_length;
219 if (thread_index >= vec_len (t->working_copies))
221 oldheap = clib_mem_set_heap (t->mheap);
222 vec_validate (t->working_copies, thread_index);
223 vec_validate (t->working_copy_lengths, thread_index);
224 t->working_copy_lengths[thread_index] = -1;
225 clib_mem_set_heap (oldheap);
229 * working_copies are per-cpu so that near-simultaneous
230 * updates from multiple threads will not result in sporadic, spurious
233 working_copy = t->working_copies[thread_index];
234 working_copy_length = t->working_copy_lengths[thread_index];
236 (sizeof(vnet_classify_entry_t) + (t->match_n_vectors*sizeof(u32x4)))
237 * t->entries_per_page * (1<<b->log2_pages);
239 t->saved_bucket.as_u64 = b->as_u64;
240 oldheap = clib_mem_set_heap (t->mheap);
242 if (required_length > working_copy_length)
245 clib_mem_free (working_copy);
247 clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
248 t->working_copies[thread_index] = working_copy;
251 clib_mem_set_heap (oldheap);
253 v = vnet_classify_get_entry (t, b->offset);
255 clib_memcpy (working_copy, v, required_length);
257 working_bucket.as_u64 = b->as_u64;
258 working_bucket.offset = vnet_classify_get_offset (t, working_copy);
259 CLIB_MEMORY_BARRIER();
260 b->as_u64 = working_bucket.as_u64;
261 t->working_copies[thread_index] = working_copy;
264 static vnet_classify_entry_t *
265 split_and_rehash (vnet_classify_table_t * t,
266 vnet_classify_entry_t * old_values, u32 old_log2_pages,
269 vnet_classify_entry_t * new_values, * v, * new_v;
270 int i, j, length_in_entries;
272 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
273 length_in_entries = (1<<old_log2_pages) * t->entries_per_page;
275 for (i = 0; i < length_in_entries; i++)
279 v = vnet_classify_entry_at_index (t, old_values, i);
281 if (vnet_classify_entry_is_busy (v))
283 /* Hack so we can use the packet hash routine */
285 key_minus_skip = (u8 *) v->key;
286 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
288 new_hash = vnet_classify_hash_packet (t, key_minus_skip);
289 new_hash >>= t->log2_nbuckets;
290 new_hash &= (1<<new_log2_pages) - 1;
292 for (j = 0; j < t->entries_per_page; j++)
294 new_v = vnet_classify_entry_at_index (t, new_values,
297 if (vnet_classify_entry_is_free (new_v))
299 clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
300 + (t->match_n_vectors * sizeof (u32x4)));
301 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
305 /* Crap. Tell caller to try again */
306 vnet_classify_entry_free (t, new_values, new_log2_pages);
315 static vnet_classify_entry_t *
316 split_and_rehash_linear (vnet_classify_table_t * t,
317 vnet_classify_entry_t * old_values,
321 vnet_classify_entry_t * new_values, * v, * new_v;
322 int i, j, new_length_in_entries, old_length_in_entries;
324 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
325 new_length_in_entries = (1<<new_log2_pages) * t->entries_per_page;
326 old_length_in_entries = (1<<old_log2_pages) * t->entries_per_page;
329 for (i = 0; i < old_length_in_entries; i++)
331 v = vnet_classify_entry_at_index (t, old_values, i);
333 if (vnet_classify_entry_is_busy (v))
335 for (; j < new_length_in_entries; j++)
337 new_v = vnet_classify_entry_at_index (t, new_values, j);
339 if (vnet_classify_entry_is_busy (new_v))
341 clib_warning ("BUG: linear rehash new entry not free!");
344 clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
345 + (t->match_n_vectors * sizeof (u32x4)));
346 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
351 * Crap. Tell caller to try again.
352 * This should never happen...
354 clib_warning ("BUG: linear rehash failed!");
355 vnet_classify_entry_free (t, new_values, new_log2_pages);
366 vnet_classify_entry_claim_resource (vnet_classify_entry_t *e)
370 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
371 fib_table_lock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
373 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
374 fib_table_lock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
380 vnet_classify_entry_release_resource (vnet_classify_entry_t *e)
384 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
385 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
387 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
388 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
393 int vnet_classify_add_del (vnet_classify_table_t * t,
394 vnet_classify_entry_t * add_v,
398 vnet_classify_bucket_t * b, tmp_b;
399 vnet_classify_entry_t * v, * new_v, * save_new_v, * working_copy, * save_v;
405 u32 old_log2_pages, new_log2_pages;
406 u32 thread_index = vlib_get_thread_index();
408 int resplit_once = 0;
409 int mark_bucket_linear;
411 ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
413 key_minus_skip = (u8 *) add_v->key;
414 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
416 hash = vnet_classify_hash_packet (t, key_minus_skip);
418 bucket_index = hash & (t->nbuckets-1);
419 b = &t->buckets[bucket_index];
421 hash >>= t->log2_nbuckets;
423 while (__sync_lock_test_and_set (t->writer_lock, 1))
426 /* First elt in the bucket? */
435 v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */);
436 clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
437 t->match_n_vectors * sizeof (u32x4));
438 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
439 vnet_classify_entry_claim_resource (v);
442 tmp_b.offset = vnet_classify_get_offset (t, v);
444 b->as_u64 = tmp_b.as_u64;
445 t->active_elements ++;
450 make_working_copy (t, b);
452 save_v = vnet_classify_get_entry (t, t->saved_bucket.offset);
453 value_index = hash & ((1<<t->saved_bucket.log2_pages)-1);
454 limit = t->entries_per_page;
455 if (PREDICT_FALSE (b->linear_search))
458 limit *= (1<<b->log2_pages);
464 * For obvious (in hindsight) reasons, see if we're supposed to
465 * replace an existing key, then look for an empty slot.
468 for (i = 0; i < limit; i++)
470 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
472 if (!memcmp (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
474 clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
475 t->match_n_vectors * sizeof(u32x4));
476 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
477 vnet_classify_entry_claim_resource (v);
479 CLIB_MEMORY_BARRIER();
480 /* Restore the previous (k,v) pairs */
481 b->as_u64 = t->saved_bucket.as_u64;
485 for (i = 0; i < limit; i++)
487 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
489 if (vnet_classify_entry_is_free (v))
491 clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
492 t->match_n_vectors * sizeof(u32x4));
493 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
494 vnet_classify_entry_claim_resource (v);
496 CLIB_MEMORY_BARRIER();
497 b->as_u64 = t->saved_bucket.as_u64;
498 t->active_elements ++;
502 /* no room at the inn... split case... */
506 for (i = 0; i < limit; i++)
508 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
510 if (!memcmp (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
512 vnet_classify_entry_release_resource (v);
513 memset (v, 0xff, sizeof (vnet_classify_entry_t) +
514 t->match_n_vectors * sizeof(u32x4));
515 v->flags |= VNET_CLASSIFY_ENTRY_FREE;
517 CLIB_MEMORY_BARRIER();
518 b->as_u64 = t->saved_bucket.as_u64;
519 t->active_elements --;
524 b->as_u64 = t->saved_bucket.as_u64;
528 old_log2_pages = t->saved_bucket.log2_pages;
529 new_log2_pages = old_log2_pages + 1;
530 working_copy = t->working_copies[thread_index];
532 if (t->saved_bucket.linear_search)
535 mark_bucket_linear = 0;
537 new_v = split_and_rehash (t, working_copy, old_log2_pages, new_log2_pages);
545 new_v = split_and_rehash (t, working_copy, old_log2_pages,
553 /* pinned collisions, use linear search */
554 new_v = split_and_rehash_linear (t, working_copy, old_log2_pages,
556 /* A new linear-search bucket? */
557 if (!t->saved_bucket.linear_search)
558 t->linear_buckets ++;
559 mark_bucket_linear = 1;
563 /* Try to add the new entry */
566 key_minus_skip = (u8 *) add_v->key;
567 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
569 new_hash = vnet_classify_hash_packet_inline (t, key_minus_skip);
570 new_hash >>= t->log2_nbuckets;
571 new_hash &= (1<<new_log2_pages) - 1;
573 limit = t->entries_per_page;
574 if (mark_bucket_linear)
576 limit *= (1<<new_log2_pages);
580 for (i = 0; i < limit; i++)
582 new_v = vnet_classify_entry_at_index (t, save_new_v, new_hash + i);
584 if (vnet_classify_entry_is_free (new_v))
586 clib_memcpy (new_v, add_v, sizeof (vnet_classify_entry_t) +
587 t->match_n_vectors * sizeof(u32x4));
588 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
589 vnet_classify_entry_claim_resource (new_v);
594 /* Crap. Try again */
595 vnet_classify_entry_free (t, save_new_v, new_log2_pages);
604 tmp_b.log2_pages = new_log2_pages;
605 tmp_b.offset = vnet_classify_get_offset (t, save_new_v);
606 tmp_b.linear_search = mark_bucket_linear;
608 CLIB_MEMORY_BARRIER();
609 b->as_u64 = tmp_b.as_u64;
610 t->active_elements ++;
611 v = vnet_classify_get_entry (t, t->saved_bucket.offset);
612 vnet_classify_entry_free (t, v, old_log2_pages);
615 CLIB_MEMORY_BARRIER();
616 t->writer_lock[0] = 0;
620 typedef CLIB_PACKED(struct {
621 ethernet_header_t eh;
623 }) classify_data_or_mask_t;
625 u64 vnet_classify_hash_packet (vnet_classify_table_t * t, u8 * h)
627 return vnet_classify_hash_packet_inline (t, h);
630 vnet_classify_entry_t *
631 vnet_classify_find_entry (vnet_classify_table_t * t,
632 u8 * h, u64 hash, f64 now)
634 return vnet_classify_find_entry_inline (t, h, hash, now);
637 static u8 * format_classify_entry (u8 * s, va_list * args)
639 vnet_classify_table_t * t = va_arg (*args, vnet_classify_table_t *);
640 vnet_classify_entry_t * e = va_arg (*args, vnet_classify_entry_t *);
643 (s, "[%u]: next_index %d advance %d opaque %d action %d metadata %d\n",
644 vnet_classify_get_offset (t, e), e->next_index, e->advance,
645 e->opaque_index, e->action, e->metadata);
648 s = format (s, " k: %U\n", format_hex_bytes, e->key,
649 t->match_n_vectors * sizeof(u32x4));
651 if (vnet_classify_entry_is_busy (e))
652 s = format (s, " hits %lld, last_heard %.2f\n",
653 e->hits, e->last_heard);
655 s = format (s, " entry is free\n");
659 u8 * format_classify_table (u8 * s, va_list * args)
661 vnet_classify_table_t * t = va_arg (*args, vnet_classify_table_t *);
662 int verbose = va_arg (*args, int);
663 vnet_classify_bucket_t * b;
664 vnet_classify_entry_t * v, * save_v;
666 u64 active_elements = 0;
668 for (i = 0; i < t->nbuckets; i++)
674 s = format (s, "[%d]: empty\n", i);
680 s = format (s, "[%d]: heap offset %d, elts %d, %s\n", i,
681 b->offset, (1<<b->log2_pages)*t->entries_per_page,
682 b->linear_search ? "LINEAR" : "normal");
685 save_v = vnet_classify_get_entry (t, b->offset);
686 for (j = 0; j < (1<<b->log2_pages); j++)
688 for (k = 0; k < t->entries_per_page; k++)
691 v = vnet_classify_entry_at_index (t, save_v,
692 j*t->entries_per_page + k);
694 if (vnet_classify_entry_is_free (v))
697 s = format (s, " %d: empty\n",
698 j * t->entries_per_page + k);
703 s = format (s, " %d: %U\n",
704 j * t->entries_per_page + k,
705 format_classify_entry, t, v);
712 s = format (s, " %lld active elements\n", active_elements);
713 s = format (s, " %d free lists\n", vec_len (t->freelists));
714 s = format (s, " %d linear-search buckets\n", t->linear_buckets);
718 int vnet_classify_add_del_table (vnet_classify_main_t * cm,
724 u32 next_table_index,
727 u8 current_data_flag,
728 i16 current_data_offset,
732 vnet_classify_table_t * t;
736 if (*table_index == ~0) /* add */
738 if (memory_size == 0)
739 return VNET_API_ERROR_INVALID_MEMORY_SIZE;
742 return VNET_API_ERROR_INVALID_VALUE;
744 t = vnet_classify_new_table (cm, mask, nbuckets, memory_size,
746 t->next_table_index = next_table_index;
747 t->miss_next_index = miss_next_index;
748 t->current_data_flag = current_data_flag;
749 t->current_data_offset = current_data_offset;
750 *table_index = t - cm->tables;
754 vnet_classify_main_t *cm = &vnet_classify_main;
755 t = pool_elt_at_index (cm->tables, *table_index);
757 t->next_table_index = next_table_index;
762 vnet_classify_delete_table_index (cm, *table_index, del_chain);
766 #define foreach_tcp_proto_field \
770 #define foreach_udp_proto_field \
774 #define foreach_ip4_proto_field \
784 uword unformat_tcp_mask (unformat_input_t * input, va_list * args)
786 u8 ** maskp = va_arg (*args, u8 **);
788 u8 found_something = 0;
792 foreach_tcp_proto_field;
795 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
798 #define _(a) else if (unformat (input, #a)) a=1;
799 foreach_tcp_proto_field
805 #define _(a) found_something += a;
806 foreach_tcp_proto_field;
809 if (found_something == 0)
812 vec_validate (mask, sizeof (*tcp) - 1);
814 tcp = (tcp_header_t *) mask;
816 #define _(a) if (a) memset (&tcp->a, 0xff, sizeof (tcp->a));
817 foreach_tcp_proto_field;
824 uword unformat_udp_mask (unformat_input_t * input, va_list * args)
826 u8 ** maskp = va_arg (*args, u8 **);
828 u8 found_something = 0;
832 foreach_udp_proto_field;
835 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
838 #define _(a) else if (unformat (input, #a)) a=1;
839 foreach_udp_proto_field
845 #define _(a) found_something += a;
846 foreach_udp_proto_field;
849 if (found_something == 0)
852 vec_validate (mask, sizeof (*udp) - 1);
854 udp = (udp_header_t *) mask;
856 #define _(a) if (a) memset (&udp->a, 0xff, sizeof (udp->a));
857 foreach_udp_proto_field;
865 u16 src_port, dst_port;
868 uword unformat_l4_mask (unformat_input_t * input, va_list * args)
870 u8 ** maskp = va_arg (*args, u8 **);
871 u16 src_port = 0, dst_port = 0;
872 tcpudp_header_t * tcpudp;
874 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
876 if (unformat (input, "tcp %U", unformat_tcp_mask, maskp))
878 else if (unformat (input, "udp %U", unformat_udp_mask, maskp))
880 else if (unformat (input, "src_port"))
882 else if (unformat (input, "dst_port"))
888 if (!src_port && !dst_port)
892 vec_validate (mask, sizeof (tcpudp_header_t) - 1);
894 tcpudp = (tcpudp_header_t *) mask;
895 tcpudp->src_port = src_port;
896 tcpudp->dst_port = dst_port;
903 uword unformat_ip4_mask (unformat_input_t * input, va_list * args)
905 u8 ** maskp = va_arg (*args, u8 **);
907 u8 found_something = 0;
911 foreach_ip4_proto_field;
917 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
919 if (unformat (input, "version"))
921 else if (unformat (input, "hdr_length"))
923 else if (unformat (input, "src"))
925 else if (unformat (input, "dst"))
927 else if (unformat (input, "proto"))
930 #define _(a) else if (unformat (input, #a)) a=1;
931 foreach_ip4_proto_field
937 #define _(a) found_something += a;
938 foreach_ip4_proto_field;
941 if (found_something == 0)
944 vec_validate (mask, sizeof (*ip) - 1);
946 ip = (ip4_header_t *) mask;
948 #define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
949 foreach_ip4_proto_field;
952 ip->ip_version_and_header_length = 0;
955 ip->ip_version_and_header_length |= 0xF0;
958 ip->ip_version_and_header_length |= 0x0F;
964 #define foreach_ip6_proto_field \
971 uword unformat_ip6_mask (unformat_input_t * input, va_list * args)
973 u8 ** maskp = va_arg (*args, u8 **);
975 u8 found_something = 0;
977 u32 ip_version_traffic_class_and_flow_label;
980 foreach_ip6_proto_field;
983 u8 traffic_class = 0;
986 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
988 if (unformat (input, "version"))
990 else if (unformat (input, "traffic-class"))
992 else if (unformat (input, "flow-label"))
994 else if (unformat (input, "src"))
996 else if (unformat (input, "dst"))
998 else if (unformat (input, "proto"))
1001 #define _(a) else if (unformat (input, #a)) a=1;
1002 foreach_ip6_proto_field
1008 #define _(a) found_something += a;
1009 foreach_ip6_proto_field;
1012 if (found_something == 0)
1015 vec_validate (mask, sizeof (*ip) - 1);
1017 ip = (ip6_header_t *) mask;
1019 #define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
1020 foreach_ip6_proto_field;
1023 ip_version_traffic_class_and_flow_label = 0;
1026 ip_version_traffic_class_and_flow_label |= 0xF0000000;
1029 ip_version_traffic_class_and_flow_label |= 0x0FF00000;
1032 ip_version_traffic_class_and_flow_label |= 0x000FFFFF;
1034 ip->ip_version_traffic_class_and_flow_label =
1035 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1041 uword unformat_l3_mask (unformat_input_t * input, va_list * args)
1043 u8 ** maskp = va_arg (*args, u8 **);
1045 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1046 if (unformat (input, "ip4 %U", unformat_ip4_mask, maskp))
1048 else if (unformat (input, "ip6 %U", unformat_ip6_mask, maskp))
1056 uword unformat_l2_mask (unformat_input_t * input, va_list * args)
1058 u8 ** maskp = va_arg (*args, u8 **);
1073 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1074 if (unformat (input, "src"))
1076 else if (unformat (input, "dst"))
1078 else if (unformat (input, "proto"))
1080 else if (unformat (input, "tag1"))
1082 else if (unformat (input, "tag2"))
1084 else if (unformat (input, "ignore-tag1"))
1086 else if (unformat (input, "ignore-tag2"))
1088 else if (unformat (input, "cos1"))
1090 else if (unformat (input, "cos2"))
1092 else if (unformat (input, "dot1q"))
1094 else if (unformat (input, "dot1ad"))
1099 if ((src + dst + proto + tag1 + tag2 + dot1q + dot1ad +
1100 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
1103 if (tag1 || ignore_tag1 || cos1 || dot1q)
1105 if (tag2 || ignore_tag2 || cos2 || dot1ad)
1108 vec_validate (mask, len-1);
1111 memset (mask, 0xff, 6);
1114 memset (mask + 6, 0xff, 6);
1118 /* inner vlan tag */
1127 mask[21] = mask [20] = 0xff;
1148 mask[16] = mask [17] = 0xff;
1157 mask[12] = mask [13] = 0xff;
1163 uword unformat_classify_mask (unformat_input_t * input, va_list * args)
1165 u8 ** maskp = va_arg (*args, u8 **);
1166 u32 * skipp = va_arg (*args, u32 *);
1167 u32 * matchp = va_arg (*args, u32 *);
1175 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1176 if (unformat (input, "hex %U", unformat_hex_string, &mask))
1178 else if (unformat (input, "l2 %U", unformat_l2_mask, &l2))
1180 else if (unformat (input, "l3 %U", unformat_l3_mask, &l3))
1182 else if (unformat (input, "l4 %U", unformat_l4_mask, &l4))
1195 if (mask || l2 || l3 || l4)
1199 /* "With a free Ethernet header in every package" */
1201 vec_validate (l2, 13);
1205 vec_append (mask, l3);
1210 vec_append (mask, l4);
1215 /* Scan forward looking for the first significant mask octet */
1216 for (i = 0; i < vec_len (mask); i++)
1220 /* compute (skip, match) params */
1221 *skipp = i / sizeof(u32x4);
1222 vec_delete (mask, *skipp * sizeof(u32x4), 0);
1224 /* Pad mask to an even multiple of the vector size */
1225 while (vec_len (mask) % sizeof (u32x4))
1228 match = vec_len (mask) / sizeof (u32x4);
1230 for (i = match*sizeof(u32x4); i > 0; i-= sizeof(u32x4))
1232 u64 *tmp = (u64 *)(mask + (i-sizeof(u32x4)));
1233 if (*tmp || *(tmp+1))
1238 clib_warning ("BUG: match 0");
1240 _vec_len (mask) = match * sizeof(u32x4);
1251 #define foreach_l2_input_next \
1253 _(ethernet, ETHERNET_INPUT) \
1258 uword unformat_l2_input_next_index (unformat_input_t * input, va_list * args)
1260 vnet_classify_main_t * cm = &vnet_classify_main;
1261 u32 * miss_next_indexp = va_arg (*args, u32 *);
1266 /* First try registered unformat fns, allowing override... */
1267 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1269 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1277 if (unformat (input, #n)) { next_index = L2_INPUT_CLASSIFY_NEXT_##N; goto out;}
1278 foreach_l2_input_next;
1281 if (unformat (input, "%d", &tmp))
1290 *miss_next_indexp = next_index;
1294 #define foreach_l2_output_next \
1297 uword unformat_l2_output_next_index (unformat_input_t * input, va_list * args)
1299 vnet_classify_main_t * cm = &vnet_classify_main;
1300 u32 * miss_next_indexp = va_arg (*args, u32 *);
1305 /* First try registered unformat fns, allowing override... */
1306 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1308 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1316 if (unformat (input, #n)) { next_index = L2_OUTPUT_CLASSIFY_NEXT_##N; goto out;}
1317 foreach_l2_output_next;
1320 if (unformat (input, "%d", &tmp))
1329 *miss_next_indexp = next_index;
1333 #define foreach_ip_next \
1337 uword unformat_ip_next_index (unformat_input_t * input, va_list * args)
1339 u32 * miss_next_indexp = va_arg (*args, u32 *);
1340 vnet_classify_main_t * cm = &vnet_classify_main;
1345 /* First try registered unformat fns, allowing override... */
1346 for (i = 0; i < vec_len (cm->unformat_ip_next_index_fns); i++)
1348 if (unformat (input, "%U", cm->unformat_ip_next_index_fns[i], &tmp))
1356 if (unformat (input, #n)) { next_index = IP_LOOKUP_NEXT_##N; goto out;}
1360 if (unformat (input, "%d", &tmp))
1369 *miss_next_indexp = next_index;
1373 #define foreach_acl_next \
1376 uword unformat_acl_next_index (unformat_input_t * input, va_list * args)
1378 u32 * next_indexp = va_arg (*args, u32 *);
1379 vnet_classify_main_t * cm = &vnet_classify_main;
1384 /* First try registered unformat fns, allowing override... */
1385 for (i = 0; i < vec_len (cm->unformat_acl_next_index_fns); i++)
1387 if (unformat (input, "%U", cm->unformat_acl_next_index_fns[i], &tmp))
1395 if (unformat (input, #n)) { next_index = ACL_NEXT_INDEX_##N; goto out;}
1399 if (unformat (input, "permit"))
1404 else if (unformat (input, "%d", &tmp))
1413 *next_indexp = next_index;
1417 uword unformat_policer_next_index (unformat_input_t * input, va_list * args)
1419 u32 * next_indexp = va_arg (*args, u32 *);
1420 vnet_classify_main_t * cm = &vnet_classify_main;
1425 /* First try registered unformat fns, allowing override... */
1426 for (i = 0; i < vec_len (cm->unformat_policer_next_index_fns); i++)
1428 if (unformat (input, "%U", cm->unformat_policer_next_index_fns[i], &tmp))
1435 if (unformat (input, "%d", &tmp))
1444 *next_indexp = next_index;
1448 static clib_error_t *
1449 classify_table_command_fn (vlib_main_t * vm,
1450 unformat_input_t * input,
1451 vlib_cli_command_t * cmd)
1458 u32 table_index = ~0;
1459 u32 next_table_index = ~0;
1460 u32 miss_next_index = ~0;
1461 u32 memory_size = 2<<20;
1463 u32 current_data_flag = 0;
1464 int current_data_offset = 0;
1467 vnet_classify_main_t * cm = &vnet_classify_main;
1470 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1471 if (unformat (input, "del"))
1473 else if (unformat (input, "del-chain"))
1478 else if (unformat (input, "buckets %d", &nbuckets))
1480 else if (unformat (input, "skip %d", &skip))
1482 else if (unformat (input, "match %d", &match))
1484 else if (unformat (input, "table %d", &table_index))
1486 else if (unformat (input, "mask %U", unformat_classify_mask,
1487 &mask, &skip, &match))
1489 else if (unformat (input, "memory-size %uM", &tmp))
1490 memory_size = tmp<<20;
1491 else if (unformat (input, "memory-size %uG", &tmp))
1492 memory_size = tmp<<30;
1493 else if (unformat (input, "next-table %d", &next_table_index))
1495 else if (unformat (input, "miss-next %U", unformat_ip_next_index,
1498 else if (unformat (input, "l2-input-miss-next %U", unformat_l2_input_next_index,
1501 else if (unformat (input, "l2-output-miss-next %U", unformat_l2_output_next_index,
1504 else if (unformat (input, "acl-miss-next %U", unformat_acl_next_index,
1507 else if (unformat (input, "current-data-flag %d", ¤t_data_flag))
1509 else if (unformat (input, "current-data-offset %d", ¤t_data_offset))
1516 if (is_add && mask == 0 && table_index == ~0)
1517 return clib_error_return (0, "Mask required");
1519 if (is_add && skip == ~0 && table_index == ~0)
1520 return clib_error_return (0, "skip count required");
1522 if (is_add && match == ~0 && table_index == ~0)
1523 return clib_error_return (0, "match count required");
1525 if (!is_add && table_index == ~0)
1526 return clib_error_return (0, "table index required for delete");
1528 rv = vnet_classify_add_del_table (cm, mask, nbuckets, memory_size,
1529 skip, match, next_table_index, miss_next_index, &table_index,
1530 current_data_flag, current_data_offset, is_add, del_chain);
1537 return clib_error_return (0, "vnet_classify_add_del_table returned %d",
1543 VLIB_CLI_COMMAND (classify_table, static) = {
1544 .path = "classify table",
1546 "classify table [miss-next|l2-miss_next|acl-miss-next <next_index>]"
1547 "\n mask <mask-value> buckets <nn> [skip <n>] [match <n>]"
1548 "\n [current-data-flag <n>] [current-data-offset <n>] [table <n>]"
1549 "\n [del] [del-chain]",
1550 .function = classify_table_command_fn,
1553 static u8 * format_vnet_classify_table (u8 * s, va_list * args)
1555 vnet_classify_main_t * cm = va_arg (*args, vnet_classify_main_t *);
1556 int verbose = va_arg (*args, int);
1557 u32 index = va_arg (*args, u32);
1558 vnet_classify_table_t * t;
1562 s = format (s, "%10s%10s%10s%10s", "TableIdx", "Sessions", "NextTbl",
1563 "NextNode", verbose ? "Details" : "");
1567 t = pool_elt_at_index (cm->tables, index);
1568 s = format (s, "%10u%10d%10d%10d", index, t->active_elements,
1569 t->next_table_index, t->miss_next_index);
1571 s = format (s, "\n Heap: %U", format_mheap, t->mheap, 0 /*verbose*/);
1573 s = format (s, "\n nbuckets %d, skip %d match %d flag %d offset %d",
1574 t->nbuckets, t->skip_n_vectors, t->match_n_vectors,
1575 t->current_data_flag, t->current_data_offset);
1576 s = format (s, "\n mask %U", format_hex_bytes, t->mask,
1577 t->match_n_vectors * sizeof (u32x4));
1578 s = format (s, "\n linear-search buckets %d\n", t->linear_buckets);
1583 s = format (s, "\n%U", format_classify_table, t, verbose);
1588 static clib_error_t *
1589 show_classify_tables_command_fn (vlib_main_t * vm,
1590 unformat_input_t * input,
1591 vlib_cli_command_t * cmd)
1593 vnet_classify_main_t * cm = &vnet_classify_main;
1594 vnet_classify_table_t * t;
1595 u32 match_index = ~0;
1600 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1602 if (unformat (input, "index %d", &match_index))
1604 else if (unformat (input, "verbose %d", &verbose))
1606 else if (unformat (input, "verbose"))
1612 pool_foreach (t, cm->tables,
1614 if (match_index == ~0 || (match_index == t - cm->tables))
1615 vec_add1 (indices, t - cm->tables);
1618 if (vec_len(indices))
1620 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm, verbose,
1622 for (i = 0; i < vec_len (indices); i++)
1623 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm,
1624 verbose, indices[i]);
1627 vlib_cli_output (vm, "No classifier tables configured");
1634 VLIB_CLI_COMMAND (show_classify_table_command, static) = {
1635 .path = "show classify tables",
1636 .short_help = "show classify tables [index <nn>]",
1637 .function = show_classify_tables_command_fn,
1640 uword unformat_l4_match (unformat_input_t * input, va_list * args)
1642 u8 ** matchp = va_arg (*args, u8 **);
1644 u8 * proto_header = 0;
1650 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1652 if (unformat (input, "src_port %d", &src_port))
1654 else if (unformat (input, "dst_port %d", &dst_port))
1660 h.src_port = clib_host_to_net_u16(src_port);
1661 h.dst_port = clib_host_to_net_u16(dst_port);
1662 vec_validate(proto_header, sizeof(h)-1);
1663 memcpy(proto_header, &h, sizeof(h));
1665 *matchp = proto_header;
1670 uword unformat_ip4_match (unformat_input_t * input, va_list * args)
1672 u8 ** matchp = va_arg (*args, u8 **);
1679 int src = 0, dst = 0;
1680 ip4_address_t src_val, dst_val;
1687 int fragment_id = 0;
1688 u32 fragment_id_val;
1694 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1696 if (unformat (input, "version %d", &version_val))
1698 else if (unformat (input, "hdr_length %d", &hdr_length_val))
1700 else if (unformat (input, "src %U", unformat_ip4_address, &src_val))
1702 else if (unformat (input, "dst %U", unformat_ip4_address, &dst_val))
1704 else if (unformat (input, "proto %d", &proto_val))
1706 else if (unformat (input, "tos %d", &tos_val))
1708 else if (unformat (input, "length %d", &length_val))
1710 else if (unformat (input, "fragment_id %d", &fragment_id_val))
1712 else if (unformat (input, "ttl %d", &ttl_val))
1714 else if (unformat (input, "checksum %d", &checksum_val))
1720 if (version + hdr_length + src + dst + proto + tos + length + fragment_id
1721 + ttl + checksum == 0)
1725 * Aligned because we use the real comparison functions
1727 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof(u32x4));
1729 ip = (ip4_header_t *) match;
1731 /* These are realistically matched in practice */
1733 ip->src_address.as_u32 = src_val.as_u32;
1736 ip->dst_address.as_u32 = dst_val.as_u32;
1739 ip->protocol = proto_val;
1742 /* These are not, but they're included for completeness */
1744 ip->ip_version_and_header_length |= (version_val & 0xF)<<4;
1747 ip->ip_version_and_header_length |= (hdr_length_val & 0xF);
1753 ip->length = clib_host_to_net_u16 (length_val);
1759 ip->checksum = clib_host_to_net_u16 (checksum_val);
1765 uword unformat_ip6_match (unformat_input_t * input, va_list * args)
1767 u8 ** matchp = va_arg (*args, u8 **);
1772 u8 traffic_class = 0;
1773 u32 traffic_class_val;
1776 int src = 0, dst = 0;
1777 ip6_address_t src_val, dst_val;
1780 int payload_length = 0;
1781 u32 payload_length_val;
1784 u32 ip_version_traffic_class_and_flow_label;
1786 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1788 if (unformat (input, "version %d", &version_val))
1790 else if (unformat (input, "traffic_class %d", &traffic_class_val))
1792 else if (unformat (input, "flow_label %d", &flow_label_val))
1794 else if (unformat (input, "src %U", unformat_ip6_address, &src_val))
1796 else if (unformat (input, "dst %U", unformat_ip6_address, &dst_val))
1798 else if (unformat (input, "proto %d", &proto_val))
1800 else if (unformat (input, "payload_length %d", &payload_length_val))
1802 else if (unformat (input, "hop_limit %d", &hop_limit_val))
1808 if (version + traffic_class + flow_label + src + dst + proto +
1809 payload_length + hop_limit == 0)
1813 * Aligned because we use the real comparison functions
1815 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof(u32x4));
1817 ip = (ip6_header_t *) match;
1820 clib_memcpy (&ip->src_address, &src_val, sizeof (ip->src_address));
1823 clib_memcpy (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
1826 ip->protocol = proto_val;
1828 ip_version_traffic_class_and_flow_label = 0;
1831 ip_version_traffic_class_and_flow_label |= (version_val & 0xF) << 28;
1834 ip_version_traffic_class_and_flow_label |= (traffic_class_val & 0xFF) << 20;
1837 ip_version_traffic_class_and_flow_label |= (flow_label_val & 0xFFFFF);
1839 ip->ip_version_traffic_class_and_flow_label =
1840 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1843 ip->payload_length = clib_host_to_net_u16 (payload_length_val);
1846 ip->hop_limit = hop_limit_val;
1852 uword unformat_l3_match (unformat_input_t * input, va_list * args)
1854 u8 ** matchp = va_arg (*args, u8 **);
1856 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1857 if (unformat (input, "ip4 %U", unformat_ip4_match, matchp))
1859 else if (unformat (input, "ip6 %U", unformat_ip6_match, matchp))
1868 uword unformat_vlan_tag (unformat_input_t * input, va_list * args)
1870 u8 * tagp = va_arg (*args, u8 *);
1873 if (unformat(input, "%d", &tag))
1875 tagp[0] = (tag>>8) & 0x0F;
1876 tagp[1] = tag & 0xFF;
1883 uword unformat_l2_match (unformat_input_t * input, va_list * args)
1885 u8 ** matchp = va_arg (*args, u8 **);
1905 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
1906 if (unformat (input, "src %U", unformat_ethernet_address, &src_val))
1908 else if (unformat (input, "dst %U", unformat_ethernet_address, &dst_val))
1910 else if (unformat (input, "proto %U",
1911 unformat_ethernet_type_host_byte_order, &proto_val))
1913 else if (unformat (input, "tag1 %U", unformat_vlan_tag, tag1_val))
1915 else if (unformat (input, "tag2 %U", unformat_vlan_tag, tag2_val))
1917 else if (unformat (input, "ignore-tag1"))
1919 else if (unformat (input, "ignore-tag2"))
1921 else if (unformat (input, "cos1 %d", &cos1_val))
1923 else if (unformat (input, "cos2 %d", &cos2_val))
1928 if ((src + dst + proto + tag1 + tag2 +
1929 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
1932 if (tag1 || ignore_tag1 || cos1)
1934 if (tag2 || ignore_tag2 || cos2)
1937 vec_validate_aligned (match, len-1, sizeof(u32x4));
1940 clib_memcpy (match, dst_val, 6);
1943 clib_memcpy (match + 6, src_val, 6);
1947 /* inner vlan tag */
1948 match[19] = tag2_val[1];
1949 match[18] = tag2_val[0];
1951 match [18] |= (cos2_val & 0x7) << 5;
1954 match[21] = proto_val & 0xff;
1955 match[20] = proto_val >> 8;
1959 match [15] = tag1_val[1];
1960 match [14] = tag1_val[0];
1963 match [14] |= (cos1_val & 0x7) << 5;
1969 match [15] = tag1_val[1];
1970 match [14] = tag1_val[0];
1973 match[17] = proto_val & 0xff;
1974 match[16] = proto_val >> 8;
1977 match [14] |= (cos1_val & 0x7) << 5;
1983 match [18] |= (cos2_val & 0x7) << 5;
1985 match [14] |= (cos1_val & 0x7) << 5;
1988 match[13] = proto_val & 0xff;
1989 match[12] = proto_val >> 8;
1997 uword unformat_classify_match (unformat_input_t * input, va_list * args)
1999 vnet_classify_main_t * cm = va_arg (*args, vnet_classify_main_t *);
2000 u8 ** matchp = va_arg (*args, u8 **);
2001 u32 table_index = va_arg (*args, u32);
2002 vnet_classify_table_t * t;
2009 if (pool_is_free_index (cm->tables, table_index))
2012 t = pool_elt_at_index (cm->tables, table_index);
2014 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2015 if (unformat (input, "hex %U", unformat_hex_string, &match))
2017 else if (unformat (input, "l2 %U", unformat_l2_match, &l2))
2019 else if (unformat (input, "l3 %U", unformat_l3_match, &l3))
2021 else if (unformat (input, "l4 %U", unformat_l4_match, &l4))
2034 if (match || l2 || l3 || l4)
2038 /* "Win a free Ethernet header in every packet" */
2040 vec_validate_aligned (l2, 13, sizeof(u32x4));
2044 vec_append_aligned (match, l3, sizeof(u32x4));
2049 vec_append_aligned (match, l4, sizeof(u32x4));
2054 /* Make sure the vector is big enough even if key is all 0's */
2055 vec_validate_aligned
2056 (match, ((t->match_n_vectors + t->skip_n_vectors) * sizeof(u32x4)) - 1,
2059 /* Set size, include skipped vectors*/
2060 _vec_len (match) = (t->match_n_vectors+t->skip_n_vectors) * sizeof(u32x4);
2070 int vnet_classify_add_del_session (vnet_classify_main_t * cm,
2080 vnet_classify_table_t * t;
2081 vnet_classify_entry_5_t _max_e __attribute__((aligned (16)));
2082 vnet_classify_entry_t * e;
2085 if (pool_is_free_index (cm->tables, table_index))
2086 return VNET_API_ERROR_NO_SUCH_TABLE;
2088 t = pool_elt_at_index (cm->tables, table_index);
2090 e = (vnet_classify_entry_t *)&_max_e;
2091 e->next_index = hit_next_index;
2092 e->opaque_index = opaque_index;
2093 e->advance = advance;
2098 if (e->action == CLASSIFY_ACTION_SET_IP4_FIB_INDEX)
2099 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2101 FIB_SOURCE_CLASSIFY);
2102 else if (e->action == CLASSIFY_ACTION_SET_IP6_FIB_INDEX)
2103 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2105 FIB_SOURCE_CLASSIFY);
2109 /* Copy key data, honoring skip_n_vectors */
2110 clib_memcpy (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
2111 t->match_n_vectors * sizeof (u32x4));
2113 /* Clear don't-care bits; likely when dynamically creating sessions */
2114 for (i = 0; i < t->match_n_vectors; i++)
2115 e->key[i] &= t->mask[i];
2117 rv = vnet_classify_add_del (t, e, is_add);
2119 vnet_classify_entry_release_resource(e);
2122 return VNET_API_ERROR_NO_SUCH_ENTRY;
2126 static clib_error_t *
2127 classify_session_command_fn (vlib_main_t * vm,
2128 unformat_input_t * input,
2129 vlib_cli_command_t * cmd)
2131 vnet_classify_main_t * cm = &vnet_classify_main;
2133 u32 table_index = ~0;
2134 u32 hit_next_index = ~0;
2135 u64 opaque_index = ~0;
2142 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2144 if (unformat (input, "del"))
2146 else if (unformat (input, "hit-next %U", unformat_ip_next_index,
2149 else if (unformat (input, "l2-input-hit-next %U", unformat_l2_input_next_index,
2152 else if (unformat (input, "l2-output-hit-next %U", unformat_l2_output_next_index,
2155 else if (unformat (input, "acl-hit-next %U", unformat_acl_next_index,
2158 else if (unformat (input, "policer-hit-next %U",
2159 unformat_policer_next_index, &hit_next_index))
2161 else if (unformat (input, "opaque-index %lld", &opaque_index))
2163 else if (unformat (input, "match %U", unformat_classify_match,
2164 cm, &match, table_index))
2166 else if (unformat (input, "advance %d", &advance))
2168 else if (unformat (input, "table-index %d", &table_index))
2170 else if (unformat (input, "action set-ip4-fib-id %d", &metadata))
2172 else if (unformat (input, "action set-ip6-fib-id %d", &metadata))
2176 /* Try registered opaque-index unformat fns */
2177 for (i = 0; i < vec_len (cm->unformat_opaque_index_fns); i++)
2179 if (unformat (input, "%U", cm->unformat_opaque_index_fns[i],
2189 if (table_index == ~0)
2190 return clib_error_return (0, "Table index required");
2192 if (is_add && match == 0)
2193 return clib_error_return (0, "Match value required");
2195 rv = vnet_classify_add_del_session (cm, table_index, match,
2197 opaque_index, advance,
2198 action, metadata, is_add);
2206 return clib_error_return (0, "vnet_classify_add_del_session returned %d",
2213 VLIB_CLI_COMMAND (classify_session_command, static) = {
2214 .path = "classify session",
2216 "classify session [hit-next|l2-hit-next|"
2217 "acl-hit-next <next_index>|policer-hit-next <policer_name>]"
2218 "\n table-index <nn> match [hex] [l2] [l3 ip4] [opaque-index <index>]"
2219 "\n [action set-ip4-fib-id <n>] [action set-ip6-fib-id <n>] [del]",
2220 .function = classify_session_command_fn,
2224 unformat_opaque_sw_if_index (unformat_input_t * input, va_list * args)
2226 u64 * opaquep = va_arg (*args, u64 *);
2229 if (unformat (input, "opaque-sw_if_index %U", unformat_vnet_sw_interface,
2230 vnet_get_main(), &sw_if_index))
2232 *opaquep = sw_if_index;
2239 unformat_ip_next_node (unformat_input_t * input, va_list * args)
2241 vnet_classify_main_t * cm = &vnet_classify_main;
2242 u32 * next_indexp = va_arg (*args, u32 *);
2244 u32 next_index = ~0;
2246 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2247 cm->vlib_main, &node_index))
2249 next_index = vlib_node_add_next (cm->vlib_main,
2250 ip6_classify_node.index, node_index);
2252 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2253 cm->vlib_main, &node_index))
2255 next_index = vlib_node_add_next (cm->vlib_main,
2256 ip4_classify_node.index, node_index);
2261 *next_indexp = next_index;
2266 unformat_acl_next_node (unformat_input_t * input, va_list * args)
2268 vnet_classify_main_t * cm = &vnet_classify_main;
2269 u32 * next_indexp = va_arg (*args, u32 *);
2273 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2274 cm->vlib_main, &node_index))
2276 next_index = vlib_node_add_next (cm->vlib_main,
2277 ip6_inacl_node.index, node_index);
2279 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2280 cm->vlib_main, &node_index))
2282 next_index = vlib_node_add_next (cm->vlib_main,
2283 ip4_inacl_node.index, node_index);
2288 *next_indexp = next_index;
2293 unformat_l2_input_next_node (unformat_input_t * input, va_list * args)
2295 vnet_classify_main_t * cm = &vnet_classify_main;
2296 u32 * next_indexp = va_arg (*args, u32 *);
2300 if (unformat (input, "input-node %U", unformat_vlib_node,
2301 cm->vlib_main, &node_index))
2303 next_index = vlib_node_add_next
2304 (cm->vlib_main, l2_input_classify_node.index, node_index);
2306 *next_indexp = next_index;
2313 unformat_l2_output_next_node (unformat_input_t * input, va_list * args)
2315 vnet_classify_main_t * cm = &vnet_classify_main;
2316 u32 * next_indexp = va_arg (*args, u32 *);
2320 if (unformat (input, "output-node %U", unformat_vlib_node,
2321 cm->vlib_main, &node_index))
2323 next_index = vlib_node_add_next
2324 (cm->vlib_main, l2_output_classify_node.index, node_index);
2326 *next_indexp = next_index;
2332 static clib_error_t *
2333 vnet_classify_init (vlib_main_t * vm)
2335 vnet_classify_main_t * cm = &vnet_classify_main;
2338 cm->vnet_main = vnet_get_main();
2340 vnet_classify_register_unformat_opaque_index_fn
2341 (unformat_opaque_sw_if_index);
2343 vnet_classify_register_unformat_ip_next_index_fn
2344 (unformat_ip_next_node);
2346 vnet_classify_register_unformat_l2_next_index_fn
2347 (unformat_l2_input_next_node);
2349 vnet_classify_register_unformat_l2_next_index_fn
2350 (unformat_l2_output_next_node);
2352 vnet_classify_register_unformat_acl_next_index_fn
2353 (unformat_acl_next_node);
2358 VLIB_INIT_FUNCTION (vnet_classify_init);
2372 test_entry_t *entries;
2374 /* test parameters */
2380 vnet_classify_table_t *table;
2388 classify_data_or_mask_t * mask;
2389 classify_data_or_mask_t * data;
2392 vnet_classify_main_t *classify_main;
2393 vlib_main_t *vlib_main;
2395 } test_classify_main_t;
2397 static test_classify_main_t test_classify_main;
2399 static clib_error_t *
2400 test_classify_churn (test_classify_main_t *tm)
2402 classify_data_or_mask_t *mask, *data;
2403 vlib_main_t *vm = tm->vlib_main;
2405 u8 *mp = 0, *dp = 0;
2409 vec_validate_aligned (mp, 3 * sizeof(u32x4), sizeof(u32x4));
2410 vec_validate_aligned (dp, 3 * sizeof(u32x4), sizeof(u32x4));
2412 mask = (classify_data_or_mask_t *) mp;
2413 data = (classify_data_or_mask_t *) dp;
2415 /* Mask on src address */
2416 memset (&mask->ip.src_address, 0xff, 4);
2418 tmp = clib_host_to_net_u32 (tm->src.as_u32);
2420 for (i = 0; i < tm->sessions; i++)
2422 vec_add2 (tm->entries, ep, 1);
2423 ep->addr.as_u32 = clib_host_to_net_u32 (tmp);
2428 tm->table = vnet_classify_new_table (tm->classify_main,
2433 3 /* vectors to match */);
2434 tm->table->miss_next_index = IP_LOOKUP_NEXT_DROP;
2435 tm->table_index = tm->table - tm->classify_main->tables;
2436 vlib_cli_output (vm, "Created table %d, buckets %d",
2437 tm->table_index, tm->buckets);
2439 vlib_cli_output (vm, "Initialize: add %d (approx. half of %d sessions)...",
2440 tm->sessions/2, tm->sessions);
2442 for (i = 0; i < tm->sessions/2; i++)
2444 ep = vec_elt_at_index (tm->entries, i);
2446 data->ip.src_address.as_u32 = ep->addr.as_u32;
2449 rv = vnet_classify_add_del_session (tm->classify_main,
2452 IP_LOOKUP_NEXT_DROP,
2453 i /* opaque_index */,
2460 clib_warning ("add: returned %d", rv);
2463 vlib_cli_output (vm, "add: %U", format_ip4_address,
2467 vlib_cli_output (vm, "Execute %d random add/delete operations",
2470 for (i = 0; i < tm->iterations; i++)
2474 /* Pick a random entry */
2475 index = random_u32 (&tm->seed) % tm->sessions;
2477 ep = vec_elt_at_index (tm->entries, index);
2479 data->ip.src_address.as_u32 = ep->addr.as_u32;
2481 /* If it's in the table, remove it. Else, add it */
2482 is_add = !ep->in_table;
2485 vlib_cli_output (vm, "%s: %U",
2486 is_add ? "add" : "del",
2490 rv = vnet_classify_add_del_session (tm->classify_main,
2493 IP_LOOKUP_NEXT_DROP,
2494 i /* opaque_index */,
2500 vlib_cli_output (vm,
2501 "%s[%d]: %U returned %d", is_add ? "add" : "del",
2504 &ep->addr.as_u32, rv);
2506 ep->in_table = is_add;
2509 vlib_cli_output (vm, "Remove remaining %d entries from the table",
2510 tm->table->active_elements);
2512 for (i = 0; i < tm->sessions; i++)
2514 u8 * key_minus_skip;
2516 vnet_classify_entry_t * e;
2518 ep = tm->entries + i;
2519 if (ep->in_table == 0)
2522 data->ip.src_address.as_u32 = ep->addr.as_u32;
2524 hash = vnet_classify_hash_packet (tm->table, (u8 *) data);
2526 e = vnet_classify_find_entry (tm->table,
2527 (u8 *) data, hash, 0 /* time_now */);
2530 clib_warning ("Couldn't find %U index %d which should be present",
2531 format_ip4_address, ep->addr, i);
2535 key_minus_skip = (u8 *)e->key;
2536 key_minus_skip -= tm->table->skip_n_vectors * sizeof (u32x4);
2538 rv = vnet_classify_add_del_session
2542 IP_LOOKUP_NEXT_DROP,
2543 i /* opaque_index */,
2544 0 /* advance */, 0, 0,
2548 clib_warning ("del: returned %d", rv);
2551 vlib_cli_output (vm, "del: %U", format_ip4_address,
2555 vlib_cli_output (vm, "%d entries remain, MUST be zero",
2556 tm->table->active_elements);
2558 vlib_cli_output (vm, "Table after cleanup: \n%U\n",
2559 format_classify_table, tm->table, 0 /* verbose */);
2564 vnet_classify_delete_table_index (tm->classify_main,
2565 tm->table_index, 1 /* del_chain */);
2567 tm->table_index = ~0;
2568 vec_free(tm->entries);
2573 static clib_error_t *
2574 test_classify_command_fn (vlib_main_t * vm,
2575 unformat_input_t * input,
2576 vlib_cli_command_t * cmd)
2578 test_classify_main_t *tm = &test_classify_main;
2579 vnet_classify_main_t * cm = &vnet_classify_main;
2582 clib_error_t * error = 0;
2585 tm->sessions = 8192;
2586 tm->iterations = 8192;
2587 tm->memory_size = 64<<20;
2588 tm->src.as_u32 = clib_net_to_host_u32 (0x0100000A);
2590 tm->seed = 0xDEADDABE;
2591 tm->classify_main = cm;
2595 /* Default starting address 1.0.0.10 */
2597 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2598 if (unformat (input, "sessions %d", &tmp))
2600 else if (unformat (input, "src %U", unformat_ip4_address, &tm->src.as_u32))
2602 else if (unformat (input, "buckets %d", &tm->buckets))
2604 else if (unformat (input, "memory-size %uM", &tmp))
2605 tm->memory_size = tmp<<20;
2606 else if (unformat (input, "memory-size %uG", &tmp))
2607 tm->memory_size = tmp<<30;
2608 else if (unformat (input, "seed %d", &tm->seed))
2610 else if (unformat (input, "verbose"))
2613 else if (unformat (input, "iterations %d", &tm->iterations))
2615 else if (unformat (input, "churn-test"))
2624 error = test_classify_churn (tm);
2627 error = clib_error_return (0, "No such test");
2634 VLIB_CLI_COMMAND (test_classify_command, static) = {
2635 .path = "test classify",
2637 "test classify [src <ip>] [sessions <nn>] [buckets <nn>] [seed <nnn>]\n"
2638 " [memory-size <nn>[M|G]]\n"
2640 .function = test_classify_command_fn,
2642 #endif /* TEST_CODE */