2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/classify/vnet_classify.h>
16 #include <vnet/classify/in_out_acl.h>
17 #include <vnet/ip/ip.h>
18 #include <vnet/api_errno.h> /* for API error numbers */
19 #include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
20 #include <vnet/fib/fib_table.h>
21 #include <vppinfra/lock.h>
25 * @brief N-tuple classifier
28 vnet_classify_main_t vnet_classify_main;
30 #if VALIDATION_SCAFFOLDING
31 /* Validation scaffolding */
33 mv (vnet_classify_table_t * t)
37 oldheap = clib_mem_set_heap (t->mheap);
39 clib_mem_set_heap (oldheap);
43 rogue (vnet_classify_table_t * t)
46 vnet_classify_entry_t *v, *save_v;
47 u32 active_elements = 0;
48 vnet_classify_bucket_t *b;
50 for (i = 0; i < t->nbuckets; i++)
55 save_v = vnet_classify_get_entry (t, b->offset);
56 for (j = 0; j < (1 << b->log2_pages); j++)
58 for (k = 0; k < t->entries_per_page; k++)
60 v = vnet_classify_entry_at_index
61 (t, save_v, j * t->entries_per_page + k);
63 if (vnet_classify_entry_is_busy (v))
69 if (active_elements != t->active_elements)
70 clib_warning ("found %u expected %u elts", active_elements,
75 mv (vnet_classify_table_t * t)
80 rogue (vnet_classify_table_t * t)
86 vnet_classify_register_unformat_l2_next_index_fn (unformat_function_t * fn)
88 vnet_classify_main_t *cm = &vnet_classify_main;
90 vec_add1 (cm->unformat_l2_next_index_fns, fn);
94 vnet_classify_register_unformat_ip_next_index_fn (unformat_function_t * fn)
96 vnet_classify_main_t *cm = &vnet_classify_main;
98 vec_add1 (cm->unformat_ip_next_index_fns, fn);
102 vnet_classify_register_unformat_acl_next_index_fn (unformat_function_t * fn)
104 vnet_classify_main_t *cm = &vnet_classify_main;
106 vec_add1 (cm->unformat_acl_next_index_fns, fn);
110 vnet_classify_register_unformat_policer_next_index_fn (unformat_function_t *
113 vnet_classify_main_t *cm = &vnet_classify_main;
115 vec_add1 (cm->unformat_policer_next_index_fns, fn);
119 vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * fn)
121 vnet_classify_main_t *cm = &vnet_classify_main;
123 vec_add1 (cm->unformat_opaque_index_fns, fn);
126 vnet_classify_table_t *
127 vnet_classify_new_table (vnet_classify_main_t * cm,
128 u8 * mask, u32 nbuckets, u32 memory_size,
129 u32 skip_n_vectors, u32 match_n_vectors)
131 vnet_classify_table_t *t;
134 nbuckets = 1 << (max_log2 (nbuckets));
136 pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
137 clib_memset (t, 0, sizeof (*t));
139 vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
140 clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4));
142 t->next_table_index = ~0;
143 t->nbuckets = nbuckets;
144 t->log2_nbuckets = max_log2 (nbuckets);
145 t->match_n_vectors = match_n_vectors;
146 t->skip_n_vectors = skip_n_vectors;
147 t->entries_per_page = 2;
149 #if USE_DLMALLOC == 0
150 t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
152 t->mheap = create_mspace (memory_size, 1 /* locked */ );
153 /* classifier requires the memory to be contiguous, so can not expand. */
154 mspace_disable_expand (t->mheap);
157 vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
158 oldheap = clib_mem_set_heap (t->mheap);
160 clib_spinlock_init (&t->writer_lock);
161 clib_mem_set_heap (oldheap);
166 vnet_classify_delete_table_index (vnet_classify_main_t * cm,
167 u32 table_index, int del_chain)
169 vnet_classify_table_t *t;
171 /* Tolerate multiple frees, up to a point */
172 if (pool_is_free_index (cm->tables, table_index))
175 t = pool_elt_at_index (cm->tables, table_index);
176 if (del_chain && t->next_table_index != ~0)
177 /* Recursively delete the entire chain */
178 vnet_classify_delete_table_index (cm, t->next_table_index, del_chain);
181 vec_free (t->buckets);
182 #if USE_DLMALLOC == 0
183 mheap_free (t->mheap);
185 destroy_mspace (t->mheap);
188 pool_put (cm->tables, t);
191 static vnet_classify_entry_t *
192 vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
194 vnet_classify_entry_t *rv = 0;
198 CLIB_SPINLOCK_ASSERT_LOCKED (&t->writer_lock);
200 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
201 * t->entries_per_page * (1 << log2_pages);
203 if (log2_pages >= vec_len (t->freelists) || t->freelists[log2_pages] == 0)
205 oldheap = clib_mem_set_heap (t->mheap);
207 vec_validate (t->freelists, log2_pages);
209 rv = clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
210 clib_mem_set_heap (oldheap);
213 rv = t->freelists[log2_pages];
214 t->freelists[log2_pages] = rv->next_free;
219 clib_memset (rv, 0xff, required_length);
224 vnet_classify_entry_free (vnet_classify_table_t * t,
225 vnet_classify_entry_t * v, u32 log2_pages)
227 CLIB_SPINLOCK_ASSERT_LOCKED (&t->writer_lock);
229 ASSERT (vec_len (t->freelists) > log2_pages);
231 v->next_free = t->freelists[log2_pages];
232 t->freelists[log2_pages] = v;
235 static inline void make_working_copy
236 (vnet_classify_table_t * t, vnet_classify_bucket_t * b)
238 vnet_classify_entry_t *v;
239 vnet_classify_bucket_t working_bucket __attribute__ ((aligned (8)));
241 vnet_classify_entry_t *working_copy;
242 u32 thread_index = vlib_get_thread_index ();
243 int working_copy_length, required_length;
245 if (thread_index >= vec_len (t->working_copies))
247 oldheap = clib_mem_set_heap (t->mheap);
248 vec_validate (t->working_copies, thread_index);
249 vec_validate (t->working_copy_lengths, thread_index);
250 t->working_copy_lengths[thread_index] = -1;
251 clib_mem_set_heap (oldheap);
255 * working_copies are per-cpu so that near-simultaneous
256 * updates from multiple threads will not result in sporadic, spurious
259 working_copy = t->working_copies[thread_index];
260 working_copy_length = t->working_copy_lengths[thread_index];
262 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
263 * t->entries_per_page * (1 << b->log2_pages);
265 t->saved_bucket.as_u64 = b->as_u64;
266 oldheap = clib_mem_set_heap (t->mheap);
268 if (required_length > working_copy_length)
271 clib_mem_free (working_copy);
273 clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
274 t->working_copies[thread_index] = working_copy;
277 clib_mem_set_heap (oldheap);
279 v = vnet_classify_get_entry (t, b->offset);
281 clib_memcpy_fast (working_copy, v, required_length);
283 working_bucket.as_u64 = b->as_u64;
284 working_bucket.offset = vnet_classify_get_offset (t, working_copy);
285 CLIB_MEMORY_BARRIER ();
286 b->as_u64 = working_bucket.as_u64;
287 t->working_copies[thread_index] = working_copy;
290 static vnet_classify_entry_t *
291 split_and_rehash (vnet_classify_table_t * t,
292 vnet_classify_entry_t * old_values, u32 old_log2_pages,
295 vnet_classify_entry_t *new_values, *v, *new_v;
296 int i, j, length_in_entries;
298 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
299 length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
301 for (i = 0; i < length_in_entries; i++)
305 v = vnet_classify_entry_at_index (t, old_values, i);
307 if (vnet_classify_entry_is_busy (v))
309 /* Hack so we can use the packet hash routine */
311 key_minus_skip = (u8 *) v->key;
312 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
314 new_hash = vnet_classify_hash_packet (t, key_minus_skip);
315 new_hash >>= t->log2_nbuckets;
316 new_hash &= (1 << new_log2_pages) - 1;
318 for (j = 0; j < t->entries_per_page; j++)
320 new_v = vnet_classify_entry_at_index (t, new_values,
323 if (vnet_classify_entry_is_free (new_v))
325 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
326 + (t->match_n_vectors * sizeof (u32x4)));
327 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
331 /* Crap. Tell caller to try again */
332 vnet_classify_entry_free (t, new_values, new_log2_pages);
341 static vnet_classify_entry_t *
342 split_and_rehash_linear (vnet_classify_table_t * t,
343 vnet_classify_entry_t * old_values,
344 u32 old_log2_pages, u32 new_log2_pages)
346 vnet_classify_entry_t *new_values, *v, *new_v;
347 int i, j, new_length_in_entries, old_length_in_entries;
349 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
350 new_length_in_entries = (1 << new_log2_pages) * t->entries_per_page;
351 old_length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
354 for (i = 0; i < old_length_in_entries; i++)
356 v = vnet_classify_entry_at_index (t, old_values, i);
358 if (vnet_classify_entry_is_busy (v))
360 for (; j < new_length_in_entries; j++)
362 new_v = vnet_classify_entry_at_index (t, new_values, j);
364 if (vnet_classify_entry_is_busy (new_v))
366 clib_warning ("BUG: linear rehash new entry not free!");
369 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
370 + (t->match_n_vectors * sizeof (u32x4)));
371 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
376 * Crap. Tell caller to try again.
377 * This should never happen...
379 clib_warning ("BUG: linear rehash failed!");
380 vnet_classify_entry_free (t, new_values, new_log2_pages);
391 vnet_classify_entry_claim_resource (vnet_classify_entry_t * e)
395 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
396 fib_table_lock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
398 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
399 fib_table_lock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
401 case CLASSIFY_ACTION_SET_METADATA:
407 vnet_classify_entry_release_resource (vnet_classify_entry_t * e)
411 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
412 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
414 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
415 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
417 case CLASSIFY_ACTION_SET_METADATA:
423 vnet_classify_add_del (vnet_classify_table_t * t,
424 vnet_classify_entry_t * add_v, int is_add)
427 vnet_classify_bucket_t *b, tmp_b;
428 vnet_classify_entry_t *v, *new_v, *save_new_v, *working_copy, *save_v;
434 u32 old_log2_pages, new_log2_pages;
435 u32 thread_index = vlib_get_thread_index ();
437 int resplit_once = 0;
438 int mark_bucket_linear;
440 ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
442 key_minus_skip = (u8 *) add_v->key;
443 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
445 hash = vnet_classify_hash_packet (t, key_minus_skip);
447 bucket_index = hash & (t->nbuckets - 1);
448 b = &t->buckets[bucket_index];
450 hash >>= t->log2_nbuckets;
452 clib_spinlock_lock (&t->writer_lock);
454 /* First elt in the bucket? */
463 v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
464 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
465 t->match_n_vectors * sizeof (u32x4));
466 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
467 vnet_classify_entry_claim_resource (v);
470 tmp_b.offset = vnet_classify_get_offset (t, v);
472 b->as_u64 = tmp_b.as_u64;
473 t->active_elements++;
478 make_working_copy (t, b);
480 save_v = vnet_classify_get_entry (t, t->saved_bucket.offset);
481 value_index = hash & ((1 << t->saved_bucket.log2_pages) - 1);
482 limit = t->entries_per_page;
483 if (PREDICT_FALSE (b->linear_search))
486 limit *= (1 << b->log2_pages);
492 * For obvious (in hindsight) reasons, see if we're supposed to
493 * replace an existing key, then look for an empty slot.
496 for (i = 0; i < limit; i++)
498 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
501 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
503 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
504 t->match_n_vectors * sizeof (u32x4));
505 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
506 vnet_classify_entry_claim_resource (v);
508 CLIB_MEMORY_BARRIER ();
509 /* Restore the previous (k,v) pairs */
510 b->as_u64 = t->saved_bucket.as_u64;
514 for (i = 0; i < limit; i++)
516 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
518 if (vnet_classify_entry_is_free (v))
520 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
521 t->match_n_vectors * sizeof (u32x4));
522 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
523 vnet_classify_entry_claim_resource (v);
525 CLIB_MEMORY_BARRIER ();
526 b->as_u64 = t->saved_bucket.as_u64;
527 t->active_elements++;
531 /* no room at the inn... split case... */
535 for (i = 0; i < limit; i++)
537 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
540 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
542 vnet_classify_entry_release_resource (v);
543 clib_memset (v, 0xff, sizeof (vnet_classify_entry_t) +
544 t->match_n_vectors * sizeof (u32x4));
545 v->flags |= VNET_CLASSIFY_ENTRY_FREE;
547 CLIB_MEMORY_BARRIER ();
548 b->as_u64 = t->saved_bucket.as_u64;
549 t->active_elements--;
554 b->as_u64 = t->saved_bucket.as_u64;
558 old_log2_pages = t->saved_bucket.log2_pages;
559 new_log2_pages = old_log2_pages + 1;
560 working_copy = t->working_copies[thread_index];
562 if (t->saved_bucket.linear_search)
565 mark_bucket_linear = 0;
567 new_v = split_and_rehash (t, working_copy, old_log2_pages, new_log2_pages);
575 new_v = split_and_rehash (t, working_copy, old_log2_pages,
583 /* pinned collisions, use linear search */
584 new_v = split_and_rehash_linear (t, working_copy, old_log2_pages,
586 /* A new linear-search bucket? */
587 if (!t->saved_bucket.linear_search)
589 mark_bucket_linear = 1;
593 /* Try to add the new entry */
596 key_minus_skip = (u8 *) add_v->key;
597 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
599 new_hash = vnet_classify_hash_packet_inline (t, key_minus_skip);
600 new_hash >>= t->log2_nbuckets;
601 new_hash &= (1 << new_log2_pages) - 1;
603 limit = t->entries_per_page;
604 if (mark_bucket_linear)
606 limit *= (1 << new_log2_pages);
610 for (i = 0; i < limit; i++)
612 new_v = vnet_classify_entry_at_index (t, save_new_v, new_hash + i);
614 if (vnet_classify_entry_is_free (new_v))
616 clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) +
617 t->match_n_vectors * sizeof (u32x4));
618 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
619 vnet_classify_entry_claim_resource (new_v);
624 /* Crap. Try again */
625 vnet_classify_entry_free (t, save_new_v, new_log2_pages);
633 tmp_b.log2_pages = new_log2_pages;
634 tmp_b.offset = vnet_classify_get_offset (t, save_new_v);
635 tmp_b.linear_search = mark_bucket_linear;
637 CLIB_MEMORY_BARRIER ();
638 b->as_u64 = tmp_b.as_u64;
639 t->active_elements++;
640 v = vnet_classify_get_entry (t, t->saved_bucket.offset);
641 vnet_classify_entry_free (t, v, old_log2_pages);
644 clib_spinlock_unlock (&t->writer_lock);
649 typedef CLIB_PACKED(struct {
650 ethernet_header_t eh;
652 }) classify_data_or_mask_t;
656 vnet_classify_hash_packet (vnet_classify_table_t * t, u8 * h)
658 return vnet_classify_hash_packet_inline (t, h);
661 vnet_classify_entry_t *
662 vnet_classify_find_entry (vnet_classify_table_t * t,
663 u8 * h, u64 hash, f64 now)
665 return vnet_classify_find_entry_inline (t, h, hash, now);
669 format_classify_entry (u8 * s, va_list * args)
671 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
672 vnet_classify_entry_t *e = va_arg (*args, vnet_classify_entry_t *);
675 (s, "[%u]: next_index %d advance %d opaque %d action %d metadata %d\n",
676 vnet_classify_get_offset (t, e), e->next_index, e->advance,
677 e->opaque_index, e->action, e->metadata);
680 s = format (s, " k: %U\n", format_hex_bytes, e->key,
681 t->match_n_vectors * sizeof (u32x4));
683 if (vnet_classify_entry_is_busy (e))
684 s = format (s, " hits %lld, last_heard %.2f\n",
685 e->hits, e->last_heard);
687 s = format (s, " entry is free\n");
692 format_classify_table (u8 * s, va_list * args)
694 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
695 int verbose = va_arg (*args, int);
696 vnet_classify_bucket_t *b;
697 vnet_classify_entry_t *v, *save_v;
699 u64 active_elements = 0;
701 for (i = 0; i < t->nbuckets; i++)
707 s = format (s, "[%d]: empty\n", i);
713 s = format (s, "[%d]: heap offset %d, elts %d, %s\n", i,
714 b->offset, (1 << b->log2_pages) * t->entries_per_page,
715 b->linear_search ? "LINEAR" : "normal");
718 save_v = vnet_classify_get_entry (t, b->offset);
719 for (j = 0; j < (1 << b->log2_pages); j++)
721 for (k = 0; k < t->entries_per_page; k++)
724 v = vnet_classify_entry_at_index (t, save_v,
725 j * t->entries_per_page + k);
727 if (vnet_classify_entry_is_free (v))
730 s = format (s, " %d: empty\n",
731 j * t->entries_per_page + k);
736 s = format (s, " %d: %U\n",
737 j * t->entries_per_page + k,
738 format_classify_entry, t, v);
745 s = format (s, " %lld active elements\n", active_elements);
746 s = format (s, " %d free lists\n", vec_len (t->freelists));
747 s = format (s, " %d linear-search buckets\n", t->linear_buckets);
752 vnet_classify_add_del_table (vnet_classify_main_t * cm,
758 u32 next_table_index,
761 u8 current_data_flag,
762 i16 current_data_offset,
763 int is_add, int del_chain)
765 vnet_classify_table_t *t;
769 if (*table_index == ~0) /* add */
771 if (memory_size == 0)
772 return VNET_API_ERROR_INVALID_MEMORY_SIZE;
775 return VNET_API_ERROR_INVALID_VALUE;
777 t = vnet_classify_new_table (cm, mask, nbuckets, memory_size,
779 t->next_table_index = next_table_index;
780 t->miss_next_index = miss_next_index;
781 t->current_data_flag = current_data_flag;
782 t->current_data_offset = current_data_offset;
783 *table_index = t - cm->tables;
787 vnet_classify_main_t *cm = &vnet_classify_main;
788 t = pool_elt_at_index (cm->tables, *table_index);
790 t->next_table_index = next_table_index;
795 vnet_classify_delete_table_index (cm, *table_index, del_chain);
799 #define foreach_tcp_proto_field \
803 #define foreach_udp_proto_field \
807 #define foreach_ip4_proto_field \
818 unformat_tcp_mask (unformat_input_t * input, va_list * args)
820 u8 **maskp = va_arg (*args, u8 **);
822 u8 found_something = 0;
826 foreach_tcp_proto_field;
829 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
832 #define _(a) else if (unformat (input, #a)) a=1;
833 foreach_tcp_proto_field
839 #define _(a) found_something += a;
840 foreach_tcp_proto_field;
843 if (found_something == 0)
846 vec_validate (mask, sizeof (*tcp) - 1);
848 tcp = (tcp_header_t *) mask;
850 #define _(a) if (a) clib_memset (&tcp->a, 0xff, sizeof (tcp->a));
851 foreach_tcp_proto_field;
859 unformat_udp_mask (unformat_input_t * input, va_list * args)
861 u8 **maskp = va_arg (*args, u8 **);
863 u8 found_something = 0;
867 foreach_udp_proto_field;
870 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
873 #define _(a) else if (unformat (input, #a)) a=1;
874 foreach_udp_proto_field
880 #define _(a) found_something += a;
881 foreach_udp_proto_field;
884 if (found_something == 0)
887 vec_validate (mask, sizeof (*udp) - 1);
889 udp = (udp_header_t *) mask;
891 #define _(a) if (a) clib_memset (&udp->a, 0xff, sizeof (udp->a));
892 foreach_udp_proto_field;
901 u16 src_port, dst_port;
905 unformat_l4_mask (unformat_input_t * input, va_list * args)
907 u8 **maskp = va_arg (*args, u8 **);
908 u16 src_port = 0, dst_port = 0;
909 tcpudp_header_t *tcpudp;
911 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
913 if (unformat (input, "tcp %U", unformat_tcp_mask, maskp))
915 else if (unformat (input, "udp %U", unformat_udp_mask, maskp))
917 else if (unformat (input, "src_port"))
919 else if (unformat (input, "dst_port"))
925 if (!src_port && !dst_port)
929 vec_validate (mask, sizeof (tcpudp_header_t) - 1);
931 tcpudp = (tcpudp_header_t *) mask;
932 tcpudp->src_port = src_port;
933 tcpudp->dst_port = dst_port;
941 unformat_ip4_mask (unformat_input_t * input, va_list * args)
943 u8 **maskp = va_arg (*args, u8 **);
945 u8 found_something = 0;
947 u32 src_prefix_len = 32;
948 u32 src_prefix_mask = ~0;
949 u32 dst_prefix_len = 32;
950 u32 dst_prefix_mask = ~0;
953 foreach_ip4_proto_field;
959 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
961 if (unformat (input, "version"))
963 else if (unformat (input, "hdr_length"))
965 else if (unformat (input, "src/%d", &src_prefix_len))
968 src_prefix_mask &= ~((1 << (32 - src_prefix_len)) - 1);
969 src_prefix_mask = clib_host_to_net_u32 (src_prefix_mask);
971 else if (unformat (input, "dst/%d", &dst_prefix_len))
974 dst_prefix_mask &= ~((1 << (32 - dst_prefix_len)) - 1);
975 dst_prefix_mask = clib_host_to_net_u32 (dst_prefix_mask);
977 else if (unformat (input, "src"))
979 else if (unformat (input, "dst"))
981 else if (unformat (input, "proto"))
984 #define _(a) else if (unformat (input, #a)) a=1;
985 foreach_ip4_proto_field
991 #define _(a) found_something += a;
992 foreach_ip4_proto_field;
995 if (found_something == 0)
998 vec_validate (mask, sizeof (*ip) - 1);
1000 ip = (ip4_header_t *) mask;
1002 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
1003 foreach_ip4_proto_field;
1007 ip->src_address.as_u32 = src_prefix_mask;
1010 ip->dst_address.as_u32 = dst_prefix_mask;
1012 ip->ip_version_and_header_length = 0;
1015 ip->ip_version_and_header_length |= 0xF0;
1018 ip->ip_version_and_header_length |= 0x0F;
1024 #define foreach_ip6_proto_field \
1032 unformat_ip6_mask (unformat_input_t * input, va_list * args)
1034 u8 **maskp = va_arg (*args, u8 **);
1036 u8 found_something = 0;
1038 u32 ip_version_traffic_class_and_flow_label;
1040 #define _(a) u8 a=0;
1041 foreach_ip6_proto_field;
1044 u8 traffic_class = 0;
1047 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1049 if (unformat (input, "version"))
1051 else if (unformat (input, "traffic-class"))
1053 else if (unformat (input, "flow-label"))
1055 else if (unformat (input, "src"))
1057 else if (unformat (input, "dst"))
1059 else if (unformat (input, "proto"))
1062 #define _(a) else if (unformat (input, #a)) a=1;
1063 foreach_ip6_proto_field
1069 #define _(a) found_something += a;
1070 foreach_ip6_proto_field;
1073 if (found_something == 0)
1076 vec_validate (mask, sizeof (*ip) - 1);
1078 ip = (ip6_header_t *) mask;
1080 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
1081 foreach_ip6_proto_field;
1084 ip_version_traffic_class_and_flow_label = 0;
1087 ip_version_traffic_class_and_flow_label |= 0xF0000000;
1090 ip_version_traffic_class_and_flow_label |= 0x0FF00000;
1093 ip_version_traffic_class_and_flow_label |= 0x000FFFFF;
1095 ip->ip_version_traffic_class_and_flow_label =
1096 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1103 unformat_l3_mask (unformat_input_t * input, va_list * args)
1105 u8 **maskp = va_arg (*args, u8 **);
1107 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1109 if (unformat (input, "ip4 %U", unformat_ip4_mask, maskp))
1111 else if (unformat (input, "ip6 %U", unformat_ip6_mask, maskp))
1120 unformat_l2_mask (unformat_input_t * input, va_list * args)
1122 u8 **maskp = va_arg (*args, u8 **);
1137 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1139 if (unformat (input, "src"))
1141 else if (unformat (input, "dst"))
1143 else if (unformat (input, "proto"))
1145 else if (unformat (input, "tag1"))
1147 else if (unformat (input, "tag2"))
1149 else if (unformat (input, "ignore-tag1"))
1151 else if (unformat (input, "ignore-tag2"))
1153 else if (unformat (input, "cos1"))
1155 else if (unformat (input, "cos2"))
1157 else if (unformat (input, "dot1q"))
1159 else if (unformat (input, "dot1ad"))
1164 if ((src + dst + proto + tag1 + tag2 + dot1q + dot1ad +
1165 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
1168 if (tag1 || ignore_tag1 || cos1 || dot1q)
1170 if (tag2 || ignore_tag2 || cos2 || dot1ad)
1173 vec_validate (mask, len - 1);
1176 clib_memset (mask, 0xff, 6);
1179 clib_memset (mask + 6, 0xff, 6);
1183 /* inner vlan tag */
1192 mask[21] = mask[20] = 0xff;
1213 mask[16] = mask[17] = 0xff;
1222 mask[12] = mask[13] = 0xff;
1229 unformat_classify_mask (unformat_input_t * input, va_list * args)
1231 u8 **maskp = va_arg (*args, u8 **);
1232 u32 *skipp = va_arg (*args, u32 *);
1233 u32 *matchp = va_arg (*args, u32 *);
1241 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1243 if (unformat (input, "hex %U", unformat_hex_string, &mask))
1245 else if (unformat (input, "l2 %U", unformat_l2_mask, &l2))
1247 else if (unformat (input, "l3 %U", unformat_l3_mask, &l3))
1249 else if (unformat (input, "l4 %U", unformat_l4_mask, &l4))
1263 if (mask || l2 || l3 || l4)
1267 /* "With a free Ethernet header in every package" */
1269 vec_validate (l2, 13);
1273 vec_append (mask, l3);
1278 vec_append (mask, l4);
1283 /* Scan forward looking for the first significant mask octet */
1284 for (i = 0; i < vec_len (mask); i++)
1288 /* compute (skip, match) params */
1289 *skipp = i / sizeof (u32x4);
1290 vec_delete (mask, *skipp * sizeof (u32x4), 0);
1292 /* Pad mask to an even multiple of the vector size */
1293 while (vec_len (mask) % sizeof (u32x4))
1296 match = vec_len (mask) / sizeof (u32x4);
1298 for (i = match * sizeof (u32x4); i > 0; i -= sizeof (u32x4))
1300 u64 *tmp = (u64 *) (mask + (i - sizeof (u32x4)));
1301 if (*tmp || *(tmp + 1))
1306 clib_warning ("BUG: match 0");
1308 _vec_len (mask) = match * sizeof (u32x4);
1319 #define foreach_l2_input_next \
1321 _(ethernet, ETHERNET_INPUT) \
1327 unformat_l2_input_next_index (unformat_input_t * input, va_list * args)
1329 vnet_classify_main_t *cm = &vnet_classify_main;
1330 u32 *miss_next_indexp = va_arg (*args, u32 *);
1335 /* First try registered unformat fns, allowing override... */
1336 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1338 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1346 if (unformat (input, #n)) { next_index = L2_INPUT_CLASSIFY_NEXT_##N; goto out;}
1347 foreach_l2_input_next;
1350 if (unformat (input, "%d", &tmp))
1359 *miss_next_indexp = next_index;
1363 #define foreach_l2_output_next \
1367 unformat_l2_output_next_index (unformat_input_t * input, va_list * args)
1369 vnet_classify_main_t *cm = &vnet_classify_main;
1370 u32 *miss_next_indexp = va_arg (*args, u32 *);
1375 /* First try registered unformat fns, allowing override... */
1376 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1378 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1386 if (unformat (input, #n)) { next_index = L2_OUTPUT_CLASSIFY_NEXT_##N; goto out;}
1387 foreach_l2_output_next;
1390 if (unformat (input, "%d", &tmp))
1399 *miss_next_indexp = next_index;
1403 #define foreach_ip_next \
1408 unformat_ip_next_index (unformat_input_t * input, va_list * args)
1410 u32 *miss_next_indexp = va_arg (*args, u32 *);
1411 vnet_classify_main_t *cm = &vnet_classify_main;
1416 /* First try registered unformat fns, allowing override... */
1417 for (i = 0; i < vec_len (cm->unformat_ip_next_index_fns); i++)
1419 if (unformat (input, "%U", cm->unformat_ip_next_index_fns[i], &tmp))
1427 if (unformat (input, #n)) { next_index = IP_LOOKUP_NEXT_##N; goto out;}
1431 if (unformat (input, "%d", &tmp))
1440 *miss_next_indexp = next_index;
1444 #define foreach_acl_next \
1448 unformat_acl_next_index (unformat_input_t * input, va_list * args)
1450 u32 *next_indexp = va_arg (*args, u32 *);
1451 vnet_classify_main_t *cm = &vnet_classify_main;
1456 /* First try registered unformat fns, allowing override... */
1457 for (i = 0; i < vec_len (cm->unformat_acl_next_index_fns); i++)
1459 if (unformat (input, "%U", cm->unformat_acl_next_index_fns[i], &tmp))
1467 if (unformat (input, #n)) { next_index = ACL_NEXT_INDEX_##N; goto out;}
1471 if (unformat (input, "permit"))
1476 else if (unformat (input, "%d", &tmp))
1485 *next_indexp = next_index;
1490 unformat_policer_next_index (unformat_input_t * input, va_list * args)
1492 u32 *next_indexp = va_arg (*args, u32 *);
1493 vnet_classify_main_t *cm = &vnet_classify_main;
1498 /* First try registered unformat fns, allowing override... */
1499 for (i = 0; i < vec_len (cm->unformat_policer_next_index_fns); i++)
1502 (input, "%U", cm->unformat_policer_next_index_fns[i], &tmp))
1509 if (unformat (input, "%d", &tmp))
1518 *next_indexp = next_index;
1522 static clib_error_t *
1523 classify_table_command_fn (vlib_main_t * vm,
1524 unformat_input_t * input, vlib_cli_command_t * cmd)
1531 u32 table_index = ~0;
1532 u32 next_table_index = ~0;
1533 u32 miss_next_index = ~0;
1534 u32 memory_size = 2 << 20;
1536 u32 current_data_flag = 0;
1537 int current_data_offset = 0;
1540 vnet_classify_main_t *cm = &vnet_classify_main;
1543 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1545 if (unformat (input, "del"))
1547 else if (unformat (input, "del-chain"))
1552 else if (unformat (input, "buckets %d", &nbuckets))
1554 else if (unformat (input, "skip %d", &skip))
1556 else if (unformat (input, "match %d", &match))
1558 else if (unformat (input, "table %d", &table_index))
1560 else if (unformat (input, "mask %U", unformat_classify_mask,
1561 &mask, &skip, &match))
1563 else if (unformat (input, "memory-size %uM", &tmp))
1564 memory_size = tmp << 20;
1565 else if (unformat (input, "memory-size %uG", &tmp))
1566 memory_size = tmp << 30;
1567 else if (unformat (input, "next-table %d", &next_table_index))
1569 else if (unformat (input, "miss-next %U", unformat_ip_next_index,
1574 (input, "l2-input-miss-next %U", unformat_l2_input_next_index,
1579 (input, "l2-output-miss-next %U", unformat_l2_output_next_index,
1582 else if (unformat (input, "acl-miss-next %U", unformat_acl_next_index,
1585 else if (unformat (input, "current-data-flag %d", ¤t_data_flag))
1588 if (unformat (input, "current-data-offset %d", ¤t_data_offset))
1595 if (is_add && mask == 0 && table_index == ~0)
1596 return clib_error_return (0, "Mask required");
1598 if (is_add && skip == ~0 && table_index == ~0)
1599 return clib_error_return (0, "skip count required");
1601 if (is_add && match == ~0 && table_index == ~0)
1602 return clib_error_return (0, "match count required");
1604 if (!is_add && table_index == ~0)
1605 return clib_error_return (0, "table index required for delete");
1607 rv = vnet_classify_add_del_table (cm, mask, nbuckets, (u32) memory_size,
1608 skip, match, next_table_index,
1609 miss_next_index, &table_index,
1610 current_data_flag, current_data_offset,
1618 return clib_error_return (0, "vnet_classify_add_del_table returned %d",
1625 VLIB_CLI_COMMAND (classify_table, static) =
1627 .path = "classify table",
1629 "classify table [miss-next|l2-miss_next|acl-miss-next <next_index>]"
1630 "\n mask <mask-value> buckets <nn> [skip <n>] [match <n>]"
1631 "\n [current-data-flag <n>] [current-data-offset <n>] [table <n>]"
1632 "\n [memory-size <nn>[M][G]] [next-table <n>]"
1633 "\n [del] [del-chain]",
1634 .function = classify_table_command_fn,
1639 filter_table_mask_compare (void *a1, void *a2)
1641 vnet_classify_main_t *cm = &vnet_classify_main;
1645 vnet_classify_table_t *t1, *t2;
1649 t1 = pool_elt_at_index (cm->tables, *ti1);
1650 t2 = pool_elt_at_index (cm->tables, *ti2);
1652 m1 = (u8 *) (t1->mask);
1653 m2 = (u8 *) (t2->mask);
1655 for (i = 0; i < vec_len (t1->mask) * sizeof (u32x4); i++)
1657 n1 += count_set_bits (m1[0]);
1661 for (i = 0; i < vec_len (t2->mask) * sizeof (u32x4); i++)
1663 n2 += count_set_bits (m2[0]);
1667 /* Reverse sort: descending number of set bits */
1676 static clib_error_t *
1677 classify_filter_command_fn (vlib_main_t * vm,
1678 unformat_input_t * input,
1679 vlib_cli_command_t * cmd)
1682 vnet_main_t *vnm = vnet_get_main ();
1683 uword memory_size = (uword) (128 << 10);
1689 u32 table_index = ~0;
1690 u32 next_table_index = ~0;
1691 u32 miss_next_index = ~0;
1692 u32 current_data_flag = 0;
1693 int current_data_offset = 0;
1695 vnet_classify_table_t *t;
1697 vnet_classify_main_t *cm = &vnet_classify_main;
1700 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1702 if (unformat (input, "del"))
1704 else if (unformat (input, "buckets %d", &nbuckets))
1706 else if (unformat (input, "mask %U", unformat_classify_mask,
1707 &mask, &skip, &match))
1709 else if (unformat (input, "memory-size %U", unformat_memory_size,
1716 if (is_add && mask == 0 && table_index == ~0)
1717 return clib_error_return (0, "Mask required");
1719 if (is_add && skip == ~0 && table_index == ~0)
1720 return clib_error_return (0, "skip count required");
1722 if (is_add && match == ~0 && table_index == ~0)
1723 return clib_error_return (0, "match count required");
1727 if (vec_len (vnm->classify_filter_table_indices) == 0)
1728 return clib_error_return (0, "No classify filter set...");
1731 table_index = vnm->classify_filter_table_indices[0];
1732 vec_reset_length (vnm->classify_filter_table_indices);
1735 /* see if we already have a table for that... */
1739 for (i = 0; i < vec_len (vnm->classify_filter_table_indices); i++)
1741 t = pool_elt_at_index (cm->tables, i);
1742 /* classifier geometry mismatch, can't use this table */
1743 if (t->match_n_vectors != match || t->skip_n_vectors != skip)
1745 /* Masks aren't congruent, can't use this table */
1746 if (vec_len (t->mask) != vec_len (mask))
1748 /* Masks aren't bit-for-bit identical, can't use this table */
1749 if (memcmp (t->mask, mask, vec_len (mask)))
1758 rv = vnet_classify_add_del_table (cm, mask, nbuckets, memory_size,
1759 skip, match, next_table_index,
1760 miss_next_index, &table_index,
1761 current_data_flag, current_data_offset,
1771 return clib_error_return (0, "vnet_classify_add_del_table returned %d",
1778 /* Remember the table */
1779 vec_add1 (vnm->classify_filter_table_indices, table_index);
1783 /* Now try to parse a session */
1784 if (unformat (input, "match %U", unformat_classify_match,
1785 cm, &match_vector, table_index) == 0)
1790 * We use hit or miss to determine whether to trace or pcap pkts
1791 * so the session setup is very limited
1793 rv = vnet_classify_add_del_session (cm, table_index,
1794 match_vector, 0 /* hit_next_index */ ,
1795 0 /* opaque_index */ ,
1801 vec_free (match_vector);
1803 /* Sort filter tables from most-specific mask to least-specific mask */
1804 vec_sort_with_function (vnm->classify_filter_table_indices,
1805 filter_table_mask_compare);
1807 ASSERT (vec_len (vnm->classify_filter_table_indices));
1809 /* Setup next_table_index fields */
1810 for (i = 0; i < vec_len (vnm->classify_filter_table_indices); i++)
1812 t = pool_elt_at_index (cm->tables,
1813 vnm->classify_filter_table_indices[i]);
1815 if ((i + 1) < vec_len (vnm->classify_filter_table_indices))
1816 t->next_table_index = vnm->classify_filter_table_indices[i + 1];
1818 t->next_table_index = ~0;
1825 * Construct an arbitrary set of packet classifier tables for use with
1826 * "pcap rx | tx trace," and (eventually) with the vpp packet
1829 * Packets which match a rule in the classifier table chain
1830 * will be traced. The tables are automatically ordered so that
1831 * matches in the most specific table are tried first.
1833 * It's reasonably likely that folks will configure a single
1834 * table with one or two matches. As a result, we configure
1835 * 8 hash buckets and 128K of match rule space. One can override
1836 * the defaults by specifiying "buckets <nnn>" and "memory-size <xxx>"
1839 * To build up complex filter chains, repeatedly issue the
1840 * classify filter debug CLI command. Each command must specify the desired
1841 * mask and match values. If a classifier table with a suitable mask
1842 * already exists, the CLI command adds a match rule to the existing table.
1843 * If not, the CLI command add a new table and the indicated mask rule
1845 * Here is a terse description of the "mask <xxx>" syntax:
1847 * l2 src dst proto tag1 tag2 ignore-tag1 ignore-tag2 cos1 cos2 dot1q dot1ad
1849 * l3 ip4 <ip4-mask> ip6 <ip6-mask>
1851 * <ip4-mask> version hdr_length src[/width] dst[/width]
1852 * tos length fragment_id ttl protocol checksum
1854 * <ip6-mask> version traffic-class flow-label src dst proto
1855 * payload_length hop_limit protocol
1857 * l4 tcp <tcp-mask> udp <udp_mask> src_port dst_port
1859 * <tcp-mask> src dst # ports
1861 * <udp-mask> src_port dst_port
1863 * To construct matches, add the values to match after the indicated keywords:
1864 * in the match syntax. For example:
1865 * mask l3 ip4 src -> match l3 ip4 src 192.168.1.11
1868 * Configuring the classify filter
1870 * Configure a simple classify filter, and configure pcap rx trace to use it:
1872 * <b><em>classify filter mask l3 ip4 src match l3 ip4 src 192.168.1.11"</em></b><br>
1873 * <b><em>pcap rx trace on max 100 filter</em></b>
1875 * Configure another fairly simple filter
1877 * <b><em>classify filter mask l3 ip4 src dst match l3 ip4 src 192.168.1.10 dst 192.168.2.10"</em></b>
1879 * Clear all current classifier filters
1881 * <b><em>classify filter del</em></b>
1883 * To inspect the classifier tables, use
1885 * <b><em>show classify table [verbose]</em></b>
1886 * The verbose form displays all of the match rules, with hit-counters
1890 VLIB_CLI_COMMAND (classify_filter, static) =
1892 .path = "classify filter",
1894 "classify filter mask <mask-value> match <match-value> [del]"
1895 "[buckets <nn>] [memory-size <n>]",
1896 .function = classify_filter_command_fn,
1901 format_vnet_classify_table (u8 * s, va_list * args)
1903 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
1904 int verbose = va_arg (*args, int);
1905 u32 index = va_arg (*args, u32);
1906 vnet_classify_table_t *t;
1910 s = format (s, "%10s%10s%10s%10s", "TableIdx", "Sessions", "NextTbl",
1911 "NextNode", verbose ? "Details" : "");
1915 t = pool_elt_at_index (cm->tables, index);
1916 s = format (s, "%10u%10d%10d%10d", index, t->active_elements,
1917 t->next_table_index, t->miss_next_index);
1919 s = format (s, "\n Heap: %U", format_mheap, t->mheap, 0 /*verbose */ );
1921 s = format (s, "\n nbuckets %d, skip %d match %d flag %d offset %d",
1922 t->nbuckets, t->skip_n_vectors, t->match_n_vectors,
1923 t->current_data_flag, t->current_data_offset);
1924 s = format (s, "\n mask %U", format_hex_bytes, t->mask,
1925 t->match_n_vectors * sizeof (u32x4));
1926 s = format (s, "\n linear-search buckets %d\n", t->linear_buckets);
1931 s = format (s, "\n%U", format_classify_table, t, verbose);
1936 static clib_error_t *
1937 show_classify_tables_command_fn (vlib_main_t * vm,
1938 unformat_input_t * input,
1939 vlib_cli_command_t * cmd)
1941 vnet_classify_main_t *cm = &vnet_classify_main;
1942 vnet_classify_table_t *t;
1943 u32 match_index = ~0;
1948 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1950 if (unformat (input, "index %d", &match_index))
1952 else if (unformat (input, "verbose %d", &verbose))
1954 else if (unformat (input, "verbose"))
1961 pool_foreach (t, cm->tables,
1963 if (match_index == ~0 || (match_index == t - cm->tables))
1964 vec_add1 (indices, t - cm->tables);
1968 if (vec_len (indices))
1970 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm, verbose,
1972 for (i = 0; i < vec_len (indices); i++)
1973 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm,
1974 verbose, indices[i]);
1977 vlib_cli_output (vm, "No classifier tables configured");
1985 VLIB_CLI_COMMAND (show_classify_table_command, static) = {
1986 .path = "show classify tables",
1987 .short_help = "show classify tables [index <nn>]",
1988 .function = show_classify_tables_command_fn,
1993 unformat_l4_match (unformat_input_t * input, va_list * args)
1995 u8 **matchp = va_arg (*args, u8 **);
1997 u8 *proto_header = 0;
2003 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2005 if (unformat (input, "src_port %d", &src_port))
2007 else if (unformat (input, "dst_port %d", &dst_port))
2013 h.src_port = clib_host_to_net_u16 (src_port);
2014 h.dst_port = clib_host_to_net_u16 (dst_port);
2015 vec_validate (proto_header, sizeof (h) - 1);
2016 memcpy (proto_header, &h, sizeof (h));
2018 *matchp = proto_header;
2024 unformat_ip4_match (unformat_input_t * input, va_list * args)
2026 u8 **matchp = va_arg (*args, u8 **);
2033 int src = 0, dst = 0;
2034 ip4_address_t src_val, dst_val;
2041 int fragment_id = 0;
2042 u32 fragment_id_val;
2048 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2050 if (unformat (input, "version %d", &version_val))
2052 else if (unformat (input, "hdr_length %d", &hdr_length_val))
2054 else if (unformat (input, "src %U", unformat_ip4_address, &src_val))
2056 else if (unformat (input, "dst %U", unformat_ip4_address, &dst_val))
2058 else if (unformat (input, "proto %d", &proto_val))
2060 else if (unformat (input, "tos %d", &tos_val))
2062 else if (unformat (input, "length %d", &length_val))
2064 else if (unformat (input, "fragment_id %d", &fragment_id_val))
2066 else if (unformat (input, "ttl %d", &ttl_val))
2068 else if (unformat (input, "checksum %d", &checksum_val))
2074 if (version + hdr_length + src + dst + proto + tos + length + fragment_id
2075 + ttl + checksum == 0)
2079 * Aligned because we use the real comparison functions
2081 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
2083 ip = (ip4_header_t *) match;
2085 /* These are realistically matched in practice */
2087 ip->src_address.as_u32 = src_val.as_u32;
2090 ip->dst_address.as_u32 = dst_val.as_u32;
2093 ip->protocol = proto_val;
2096 /* These are not, but they're included for completeness */
2098 ip->ip_version_and_header_length |= (version_val & 0xF) << 4;
2101 ip->ip_version_and_header_length |= (hdr_length_val & 0xF);
2107 ip->length = clib_host_to_net_u16 (length_val);
2113 ip->checksum = clib_host_to_net_u16 (checksum_val);
2120 unformat_ip6_match (unformat_input_t * input, va_list * args)
2122 u8 **matchp = va_arg (*args, u8 **);
2127 u8 traffic_class = 0;
2128 u32 traffic_class_val;
2131 int src = 0, dst = 0;
2132 ip6_address_t src_val, dst_val;
2135 int payload_length = 0;
2136 u32 payload_length_val;
2139 u32 ip_version_traffic_class_and_flow_label;
2141 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2143 if (unformat (input, "version %d", &version_val))
2145 else if (unformat (input, "traffic_class %d", &traffic_class_val))
2147 else if (unformat (input, "flow_label %d", &flow_label_val))
2149 else if (unformat (input, "src %U", unformat_ip6_address, &src_val))
2151 else if (unformat (input, "dst %U", unformat_ip6_address, &dst_val))
2153 else if (unformat (input, "proto %d", &proto_val))
2155 else if (unformat (input, "payload_length %d", &payload_length_val))
2157 else if (unformat (input, "hop_limit %d", &hop_limit_val))
2163 if (version + traffic_class + flow_label + src + dst + proto +
2164 payload_length + hop_limit == 0)
2168 * Aligned because we use the real comparison functions
2170 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
2172 ip = (ip6_header_t *) match;
2175 clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address));
2178 clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
2181 ip->protocol = proto_val;
2183 ip_version_traffic_class_and_flow_label = 0;
2186 ip_version_traffic_class_and_flow_label |= (version_val & 0xF) << 28;
2189 ip_version_traffic_class_and_flow_label |=
2190 (traffic_class_val & 0xFF) << 20;
2193 ip_version_traffic_class_and_flow_label |= (flow_label_val & 0xFFFFF);
2195 ip->ip_version_traffic_class_and_flow_label =
2196 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
2199 ip->payload_length = clib_host_to_net_u16 (payload_length_val);
2202 ip->hop_limit = hop_limit_val;
2209 unformat_l3_match (unformat_input_t * input, va_list * args)
2211 u8 **matchp = va_arg (*args, u8 **);
2213 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2215 if (unformat (input, "ip4 %U", unformat_ip4_match, matchp))
2217 else if (unformat (input, "ip6 %U", unformat_ip6_match, matchp))
2227 unformat_vlan_tag (unformat_input_t * input, va_list * args)
2229 u8 *tagp = va_arg (*args, u8 *);
2232 if (unformat (input, "%d", &tag))
2234 tagp[0] = (tag >> 8) & 0x0F;
2235 tagp[1] = tag & 0xFF;
2243 unformat_l2_match (unformat_input_t * input, va_list * args)
2245 u8 **matchp = va_arg (*args, u8 **);
2265 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2267 if (unformat (input, "src %U", unformat_ethernet_address, &src_val))
2270 if (unformat (input, "dst %U", unformat_ethernet_address, &dst_val))
2272 else if (unformat (input, "proto %U",
2273 unformat_ethernet_type_host_byte_order, &proto_val))
2275 else if (unformat (input, "tag1 %U", unformat_vlan_tag, tag1_val))
2277 else if (unformat (input, "tag2 %U", unformat_vlan_tag, tag2_val))
2279 else if (unformat (input, "ignore-tag1"))
2281 else if (unformat (input, "ignore-tag2"))
2283 else if (unformat (input, "cos1 %d", &cos1_val))
2285 else if (unformat (input, "cos2 %d", &cos2_val))
2290 if ((src + dst + proto + tag1 + tag2 +
2291 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
2294 if (tag1 || ignore_tag1 || cos1)
2296 if (tag2 || ignore_tag2 || cos2)
2299 vec_validate_aligned (match, len - 1, sizeof (u32x4));
2302 clib_memcpy_fast (match, dst_val, 6);
2305 clib_memcpy_fast (match + 6, src_val, 6);
2309 /* inner vlan tag */
2310 match[19] = tag2_val[1];
2311 match[18] = tag2_val[0];
2313 match[18] |= (cos2_val & 0x7) << 5;
2316 match[21] = proto_val & 0xff;
2317 match[20] = proto_val >> 8;
2321 match[15] = tag1_val[1];
2322 match[14] = tag1_val[0];
2325 match[14] |= (cos1_val & 0x7) << 5;
2331 match[15] = tag1_val[1];
2332 match[14] = tag1_val[0];
2335 match[17] = proto_val & 0xff;
2336 match[16] = proto_val >> 8;
2339 match[14] |= (cos1_val & 0x7) << 5;
2345 match[18] |= (cos2_val & 0x7) << 5;
2347 match[14] |= (cos1_val & 0x7) << 5;
2350 match[13] = proto_val & 0xff;
2351 match[12] = proto_val >> 8;
2360 unformat_classify_match (unformat_input_t * input, va_list * args)
2362 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
2363 u8 **matchp = va_arg (*args, u8 **);
2364 u32 table_index = va_arg (*args, u32);
2365 vnet_classify_table_t *t;
2372 if (pool_is_free_index (cm->tables, table_index))
2375 t = pool_elt_at_index (cm->tables, table_index);
2377 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2379 if (unformat (input, "hex %U", unformat_hex_string, &match))
2381 else if (unformat (input, "l2 %U", unformat_l2_match, &l2))
2383 else if (unformat (input, "l3 %U", unformat_l3_match, &l3))
2385 else if (unformat (input, "l4 %U", unformat_l4_match, &l4))
2399 if (match || l2 || l3 || l4)
2403 /* "Win a free Ethernet header in every packet" */
2405 vec_validate_aligned (l2, 13, sizeof (u32x4));
2409 vec_append_aligned (match, l3, sizeof (u32x4));
2414 vec_append_aligned (match, l4, sizeof (u32x4));
2419 /* Make sure the vector is big enough even if key is all 0's */
2420 vec_validate_aligned
2422 ((t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4)) - 1,
2425 /* Set size, include skipped vectors */
2427 (t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4);
2438 vnet_classify_add_del_session (vnet_classify_main_t * cm,
2444 u8 action, u32 metadata, int is_add)
2446 vnet_classify_table_t *t;
2447 vnet_classify_entry_5_t _max_e __attribute__ ((aligned (16)));
2448 vnet_classify_entry_t *e;
2451 if (pool_is_free_index (cm->tables, table_index))
2452 return VNET_API_ERROR_NO_SUCH_TABLE;
2454 t = pool_elt_at_index (cm->tables, table_index);
2456 e = (vnet_classify_entry_t *) & _max_e;
2457 e->next_index = hit_next_index;
2458 e->opaque_index = opaque_index;
2459 e->advance = advance;
2464 if (e->action == CLASSIFY_ACTION_SET_IP4_FIB_INDEX)
2465 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2467 FIB_SOURCE_CLASSIFY);
2468 else if (e->action == CLASSIFY_ACTION_SET_IP6_FIB_INDEX)
2469 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2471 FIB_SOURCE_CLASSIFY);
2472 else if (e->action == CLASSIFY_ACTION_SET_METADATA)
2473 e->metadata = metadata;
2477 /* Copy key data, honoring skip_n_vectors */
2478 clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
2479 t->match_n_vectors * sizeof (u32x4));
2481 /* Clear don't-care bits; likely when dynamically creating sessions */
2482 for (i = 0; i < t->match_n_vectors; i++)
2483 e->key[i] &= t->mask[i];
2485 rv = vnet_classify_add_del (t, e, is_add);
2487 vnet_classify_entry_release_resource (e);
2490 return VNET_API_ERROR_NO_SUCH_ENTRY;
2494 static clib_error_t *
2495 classify_session_command_fn (vlib_main_t * vm,
2496 unformat_input_t * input,
2497 vlib_cli_command_t * cmd)
2499 vnet_classify_main_t *cm = &vnet_classify_main;
2501 u32 table_index = ~0;
2502 u32 hit_next_index = ~0;
2503 u64 opaque_index = ~0;
2510 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2512 if (unformat (input, "del"))
2514 else if (unformat (input, "hit-next %U", unformat_ip_next_index,
2519 (input, "l2-input-hit-next %U", unformat_l2_input_next_index,
2524 (input, "l2-output-hit-next %U", unformat_l2_output_next_index,
2527 else if (unformat (input, "acl-hit-next %U", unformat_acl_next_index,
2530 else if (unformat (input, "policer-hit-next %U",
2531 unformat_policer_next_index, &hit_next_index))
2533 else if (unformat (input, "opaque-index %lld", &opaque_index))
2535 else if (unformat (input, "match %U", unformat_classify_match,
2536 cm, &match, table_index))
2538 else if (unformat (input, "advance %d", &advance))
2540 else if (unformat (input, "table-index %d", &table_index))
2542 else if (unformat (input, "action set-ip4-fib-id %d", &metadata))
2544 else if (unformat (input, "action set-ip6-fib-id %d", &metadata))
2546 else if (unformat (input, "action set-sr-policy-index %d", &metadata))
2550 /* Try registered opaque-index unformat fns */
2551 for (i = 0; i < vec_len (cm->unformat_opaque_index_fns); i++)
2553 if (unformat (input, "%U", cm->unformat_opaque_index_fns[i],
2563 if (table_index == ~0)
2564 return clib_error_return (0, "Table index required");
2566 if (is_add && match == 0)
2567 return clib_error_return (0, "Match value required");
2569 rv = vnet_classify_add_del_session (cm, table_index, match,
2571 opaque_index, advance,
2572 action, metadata, is_add);
2580 return clib_error_return (0,
2581 "vnet_classify_add_del_session returned %d",
2589 VLIB_CLI_COMMAND (classify_session_command, static) = {
2590 .path = "classify session",
2592 "classify session [hit-next|l2-input-hit-next|l2-output-hit-next|"
2593 "acl-hit-next <next_index>|policer-hit-next <policer_name>]"
2594 "\n table-index <nn> match [hex] [l2] [l3 ip4] [opaque-index <index>]"
2595 "\n [action set-ip4-fib-id|set-ip6-fib-id|set-sr-policy-index <n>] [del]",
2596 .function = classify_session_command_fn,
2601 unformat_opaque_sw_if_index (unformat_input_t * input, va_list * args)
2603 u64 *opaquep = va_arg (*args, u64 *);
2606 if (unformat (input, "opaque-sw_if_index %U", unformat_vnet_sw_interface,
2607 vnet_get_main (), &sw_if_index))
2609 *opaquep = sw_if_index;
2616 unformat_ip_next_node (unformat_input_t * input, va_list * args)
2618 vnet_classify_main_t *cm = &vnet_classify_main;
2619 u32 *next_indexp = va_arg (*args, u32 *);
2621 u32 next_index = ~0;
2623 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2624 cm->vlib_main, &node_index))
2626 next_index = vlib_node_add_next (cm->vlib_main,
2627 ip6_classify_node.index, node_index);
2629 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2630 cm->vlib_main, &node_index))
2632 next_index = vlib_node_add_next (cm->vlib_main,
2633 ip4_classify_node.index, node_index);
2638 *next_indexp = next_index;
2643 unformat_acl_next_node (unformat_input_t * input, va_list * args)
2645 vnet_classify_main_t *cm = &vnet_classify_main;
2646 u32 *next_indexp = va_arg (*args, u32 *);
2650 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2651 cm->vlib_main, &node_index))
2653 next_index = vlib_node_add_next (cm->vlib_main,
2654 ip6_inacl_node.index, node_index);
2656 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2657 cm->vlib_main, &node_index))
2659 next_index = vlib_node_add_next (cm->vlib_main,
2660 ip4_inacl_node.index, node_index);
2665 *next_indexp = next_index;
2670 unformat_l2_input_next_node (unformat_input_t * input, va_list * args)
2672 vnet_classify_main_t *cm = &vnet_classify_main;
2673 u32 *next_indexp = va_arg (*args, u32 *);
2677 if (unformat (input, "input-node %U", unformat_vlib_node,
2678 cm->vlib_main, &node_index))
2680 next_index = vlib_node_add_next
2681 (cm->vlib_main, l2_input_classify_node.index, node_index);
2683 *next_indexp = next_index;
2690 unformat_l2_output_next_node (unformat_input_t * input, va_list * args)
2692 vnet_classify_main_t *cm = &vnet_classify_main;
2693 u32 *next_indexp = va_arg (*args, u32 *);
2697 if (unformat (input, "output-node %U", unformat_vlib_node,
2698 cm->vlib_main, &node_index))
2700 next_index = vlib_node_add_next
2701 (cm->vlib_main, l2_output_classify_node.index, node_index);
2703 *next_indexp = next_index;
2709 static clib_error_t *
2710 vnet_classify_init (vlib_main_t * vm)
2712 vnet_classify_main_t *cm = &vnet_classify_main;
2715 cm->vnet_main = vnet_get_main ();
2717 vnet_classify_register_unformat_opaque_index_fn
2718 (unformat_opaque_sw_if_index);
2720 vnet_classify_register_unformat_ip_next_index_fn (unformat_ip_next_node);
2722 vnet_classify_register_unformat_l2_next_index_fn
2723 (unformat_l2_input_next_node);
2725 vnet_classify_register_unformat_l2_next_index_fn
2726 (unformat_l2_output_next_node);
2728 vnet_classify_register_unformat_acl_next_index_fn (unformat_acl_next_node);
2733 VLIB_INIT_FUNCTION (vnet_classify_init);
2747 test_entry_t *entries;
2749 /* test parameters */
2755 vnet_classify_table_t *table;
2763 classify_data_or_mask_t *mask;
2764 classify_data_or_mask_t *data;
2767 vnet_classify_main_t *classify_main;
2768 vlib_main_t *vlib_main;
2770 } test_classify_main_t;
2772 static test_classify_main_t test_classify_main;
2774 static clib_error_t *
2775 test_classify_churn (test_classify_main_t * tm)
2777 classify_data_or_mask_t *mask, *data;
2778 vlib_main_t *vm = tm->vlib_main;
2780 u8 *mp = 0, *dp = 0;
2784 vec_validate_aligned (mp, 3 * sizeof (u32x4), sizeof (u32x4));
2785 vec_validate_aligned (dp, 3 * sizeof (u32x4), sizeof (u32x4));
2787 mask = (classify_data_or_mask_t *) mp;
2788 data = (classify_data_or_mask_t *) dp;
2790 /* Mask on src address */
2791 clib_memset (&mask->ip.src_address, 0xff, 4);
2793 tmp = clib_host_to_net_u32 (tm->src.as_u32);
2795 for (i = 0; i < tm->sessions; i++)
2797 vec_add2 (tm->entries, ep, 1);
2798 ep->addr.as_u32 = clib_host_to_net_u32 (tmp);
2803 tm->table = vnet_classify_new_table (tm->classify_main,
2806 tm->memory_size, 0 /* skip */ ,
2807 3 /* vectors to match */ );
2808 tm->table->miss_next_index = IP_LOOKUP_NEXT_DROP;
2809 tm->table_index = tm->table - tm->classify_main->tables;
2810 vlib_cli_output (vm, "Created table %d, buckets %d",
2811 tm->table_index, tm->buckets);
2813 vlib_cli_output (vm, "Initialize: add %d (approx. half of %d sessions)...",
2814 tm->sessions / 2, tm->sessions);
2816 for (i = 0; i < tm->sessions / 2; i++)
2818 ep = vec_elt_at_index (tm->entries, i);
2820 data->ip.src_address.as_u32 = ep->addr.as_u32;
2823 rv = vnet_classify_add_del_session (tm->classify_main,
2826 IP_LOOKUP_NEXT_DROP,
2827 i /* opaque_index */ ,
2834 clib_warning ("add: returned %d", rv);
2837 vlib_cli_output (vm, "add: %U", format_ip4_address, &ep->addr.as_u32);
2840 vlib_cli_output (vm, "Execute %d random add/delete operations",
2843 for (i = 0; i < tm->iterations; i++)
2847 /* Pick a random entry */
2848 index = random_u32 (&tm->seed) % tm->sessions;
2850 ep = vec_elt_at_index (tm->entries, index);
2852 data->ip.src_address.as_u32 = ep->addr.as_u32;
2854 /* If it's in the table, remove it. Else, add it */
2855 is_add = !ep->in_table;
2858 vlib_cli_output (vm, "%s: %U",
2859 is_add ? "add" : "del",
2860 format_ip4_address, &ep->addr.as_u32);
2862 rv = vnet_classify_add_del_session (tm->classify_main,
2865 IP_LOOKUP_NEXT_DROP,
2866 i /* opaque_index */ ,
2872 vlib_cli_output (vm,
2873 "%s[%d]: %U returned %d", is_add ? "add" : "del",
2874 index, format_ip4_address, &ep->addr.as_u32, rv);
2876 ep->in_table = is_add;
2879 vlib_cli_output (vm, "Remove remaining %d entries from the table",
2880 tm->table->active_elements);
2882 for (i = 0; i < tm->sessions; i++)
2886 vnet_classify_entry_t *e;
2888 ep = tm->entries + i;
2889 if (ep->in_table == 0)
2892 data->ip.src_address.as_u32 = ep->addr.as_u32;
2894 hash = vnet_classify_hash_packet (tm->table, (u8 *) data);
2896 e = vnet_classify_find_entry (tm->table,
2897 (u8 *) data, hash, 0 /* time_now */ );
2900 clib_warning ("Couldn't find %U index %d which should be present",
2901 format_ip4_address, ep->addr, i);
2905 key_minus_skip = (u8 *) e->key;
2906 key_minus_skip -= tm->table->skip_n_vectors * sizeof (u32x4);
2908 rv = vnet_classify_add_del_session
2911 key_minus_skip, IP_LOOKUP_NEXT_DROP, i /* opaque_index */ ,
2912 0 /* advance */ , 0, 0,
2916 clib_warning ("del: returned %d", rv);
2919 vlib_cli_output (vm, "del: %U", format_ip4_address, &ep->addr.as_u32);
2922 vlib_cli_output (vm, "%d entries remain, MUST be zero",
2923 tm->table->active_elements);
2925 vlib_cli_output (vm, "Table after cleanup: \n%U\n",
2926 format_classify_table, tm->table, 0 /* verbose */ );
2931 vnet_classify_delete_table_index (tm->classify_main,
2932 tm->table_index, 1 /* del_chain */ );
2934 tm->table_index = ~0;
2935 vec_free (tm->entries);
2940 static clib_error_t *
2941 test_classify_command_fn (vlib_main_t * vm,
2942 unformat_input_t * input, vlib_cli_command_t * cmd)
2944 test_classify_main_t *tm = &test_classify_main;
2945 vnet_classify_main_t *cm = &vnet_classify_main;
2948 clib_error_t *error = 0;
2951 tm->sessions = 8192;
2952 tm->iterations = 8192;
2953 tm->memory_size = 64 << 20;
2954 tm->src.as_u32 = clib_net_to_host_u32 (0x0100000A);
2956 tm->seed = 0xDEADDABE;
2957 tm->classify_main = cm;
2961 /* Default starting address 1.0.0.10 */
2963 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2965 if (unformat (input, "sessions %d", &tmp))
2968 if (unformat (input, "src %U", unformat_ip4_address, &tm->src.as_u32))
2970 else if (unformat (input, "buckets %d", &tm->buckets))
2972 else if (unformat (input, "memory-size %uM", &tmp))
2973 tm->memory_size = tmp << 20;
2974 else if (unformat (input, "memory-size %uG", &tmp))
2975 tm->memory_size = tmp << 30;
2976 else if (unformat (input, "seed %d", &tm->seed))
2978 else if (unformat (input, "verbose"))
2981 else if (unformat (input, "iterations %d", &tm->iterations))
2983 else if (unformat (input, "churn-test"))
2992 error = test_classify_churn (tm);
2995 error = clib_error_return (0, "No such test");
3003 VLIB_CLI_COMMAND (test_classify_command, static) = {
3004 .path = "test classify",
3006 "test classify [src <ip>] [sessions <nn>] [buckets <nn>] [seed <nnn>]\n"
3007 " [memory-size <nn>[M|G]]\n"
3009 .function = test_classify_command_fn,
3012 #endif /* TEST_CODE */
3015 * fd.io coding-style-patch-verification: ON
3018 * eval: (c-set-style "gnu")