2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/classify/vnet_classify.h>
16 #include <vnet/classify/in_out_acl.h>
17 #include <vnet/ip/ip.h>
18 #include <vnet/api_errno.h> /* for API error numbers */
19 #include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
20 #include <vnet/fib/fib_table.h>
22 vnet_classify_main_t vnet_classify_main;
24 #if VALIDATION_SCAFFOLDING
25 /* Validation scaffolding */
27 mv (vnet_classify_table_t * t)
31 oldheap = clib_mem_set_heap (t->mheap);
33 clib_mem_set_heap (oldheap);
37 rogue (vnet_classify_table_t * t)
40 vnet_classify_entry_t *v, *save_v;
41 u32 active_elements = 0;
42 vnet_classify_bucket_t *b;
44 for (i = 0; i < t->nbuckets; i++)
49 save_v = vnet_classify_get_entry (t, b->offset);
50 for (j = 0; j < (1 << b->log2_pages); j++)
52 for (k = 0; k < t->entries_per_page; k++)
54 v = vnet_classify_entry_at_index
55 (t, save_v, j * t->entries_per_page + k);
57 if (vnet_classify_entry_is_busy (v))
63 if (active_elements != t->active_elements)
64 clib_warning ("found %u expected %u elts", active_elements,
69 mv (vnet_classify_table_t * t)
74 rogue (vnet_classify_table_t * t)
80 vnet_classify_register_unformat_l2_next_index_fn (unformat_function_t * fn)
82 vnet_classify_main_t *cm = &vnet_classify_main;
84 vec_add1 (cm->unformat_l2_next_index_fns, fn);
88 vnet_classify_register_unformat_ip_next_index_fn (unformat_function_t * fn)
90 vnet_classify_main_t *cm = &vnet_classify_main;
92 vec_add1 (cm->unformat_ip_next_index_fns, fn);
96 vnet_classify_register_unformat_acl_next_index_fn (unformat_function_t * fn)
98 vnet_classify_main_t *cm = &vnet_classify_main;
100 vec_add1 (cm->unformat_acl_next_index_fns, fn);
104 vnet_classify_register_unformat_policer_next_index_fn (unformat_function_t *
107 vnet_classify_main_t *cm = &vnet_classify_main;
109 vec_add1 (cm->unformat_policer_next_index_fns, fn);
113 vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * fn)
115 vnet_classify_main_t *cm = &vnet_classify_main;
117 vec_add1 (cm->unformat_opaque_index_fns, fn);
120 vnet_classify_table_t *
121 vnet_classify_new_table (vnet_classify_main_t * cm,
122 u8 * mask, u32 nbuckets, u32 memory_size,
123 u32 skip_n_vectors, u32 match_n_vectors)
125 vnet_classify_table_t *t;
128 nbuckets = 1 << (max_log2 (nbuckets));
130 pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
131 clib_memset (t, 0, sizeof (*t));
133 vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
134 clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4));
136 t->next_table_index = ~0;
137 t->nbuckets = nbuckets;
138 t->log2_nbuckets = max_log2 (nbuckets);
139 t->match_n_vectors = match_n_vectors;
140 t->skip_n_vectors = skip_n_vectors;
141 t->entries_per_page = 2;
143 #if USE_DLMALLOC == 0
144 t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
146 t->mheap = create_mspace (memory_size, 1 /* locked */ );
147 /* classifier requires the memory to be contiguous, so can not expand. */
148 mspace_disable_expand (t->mheap);
151 vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
152 oldheap = clib_mem_set_heap (t->mheap);
154 t->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
155 CLIB_CACHE_LINE_BYTES);
156 t->writer_lock[0] = 0;
158 clib_mem_set_heap (oldheap);
163 vnet_classify_delete_table_index (vnet_classify_main_t * cm,
164 u32 table_index, int del_chain)
166 vnet_classify_table_t *t;
168 /* Tolerate multiple frees, up to a point */
169 if (pool_is_free_index (cm->tables, table_index))
172 t = pool_elt_at_index (cm->tables, table_index);
173 if (del_chain && t->next_table_index != ~0)
174 /* Recursively delete the entire chain */
175 vnet_classify_delete_table_index (cm, t->next_table_index, del_chain);
178 vec_free (t->buckets);
179 #if USE_DLMALLOC == 0
180 mheap_free (t->mheap);
182 destroy_mspace (t->mheap);
185 pool_put (cm->tables, t);
188 static vnet_classify_entry_t *
189 vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
191 vnet_classify_entry_t *rv = 0;
195 ASSERT (t->writer_lock[0]);
197 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
198 * t->entries_per_page * (1 << log2_pages);
200 if (log2_pages >= vec_len (t->freelists) || t->freelists[log2_pages] == 0)
202 oldheap = clib_mem_set_heap (t->mheap);
204 vec_validate (t->freelists, log2_pages);
206 rv = clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
207 clib_mem_set_heap (oldheap);
210 rv = t->freelists[log2_pages];
211 t->freelists[log2_pages] = rv->next_free;
216 clib_memset (rv, 0xff, required_length);
221 vnet_classify_entry_free (vnet_classify_table_t * t,
222 vnet_classify_entry_t * v, u32 log2_pages)
224 ASSERT (t->writer_lock[0]);
226 ASSERT (vec_len (t->freelists) > log2_pages);
228 v->next_free = t->freelists[log2_pages];
229 t->freelists[log2_pages] = v;
232 static inline void make_working_copy
233 (vnet_classify_table_t * t, vnet_classify_bucket_t * b)
235 vnet_classify_entry_t *v;
236 vnet_classify_bucket_t working_bucket __attribute__ ((aligned (8)));
238 vnet_classify_entry_t *working_copy;
239 u32 thread_index = vlib_get_thread_index ();
240 int working_copy_length, required_length;
242 if (thread_index >= vec_len (t->working_copies))
244 oldheap = clib_mem_set_heap (t->mheap);
245 vec_validate (t->working_copies, thread_index);
246 vec_validate (t->working_copy_lengths, thread_index);
247 t->working_copy_lengths[thread_index] = -1;
248 clib_mem_set_heap (oldheap);
252 * working_copies are per-cpu so that near-simultaneous
253 * updates from multiple threads will not result in sporadic, spurious
256 working_copy = t->working_copies[thread_index];
257 working_copy_length = t->working_copy_lengths[thread_index];
259 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
260 * t->entries_per_page * (1 << b->log2_pages);
262 t->saved_bucket.as_u64 = b->as_u64;
263 oldheap = clib_mem_set_heap (t->mheap);
265 if (required_length > working_copy_length)
268 clib_mem_free (working_copy);
270 clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
271 t->working_copies[thread_index] = working_copy;
274 clib_mem_set_heap (oldheap);
276 v = vnet_classify_get_entry (t, b->offset);
278 clib_memcpy_fast (working_copy, v, required_length);
280 working_bucket.as_u64 = b->as_u64;
281 working_bucket.offset = vnet_classify_get_offset (t, working_copy);
282 CLIB_MEMORY_BARRIER ();
283 b->as_u64 = working_bucket.as_u64;
284 t->working_copies[thread_index] = working_copy;
287 static vnet_classify_entry_t *
288 split_and_rehash (vnet_classify_table_t * t,
289 vnet_classify_entry_t * old_values, u32 old_log2_pages,
292 vnet_classify_entry_t *new_values, *v, *new_v;
293 int i, j, length_in_entries;
295 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
296 length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
298 for (i = 0; i < length_in_entries; i++)
302 v = vnet_classify_entry_at_index (t, old_values, i);
304 if (vnet_classify_entry_is_busy (v))
306 /* Hack so we can use the packet hash routine */
308 key_minus_skip = (u8 *) v->key;
309 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
311 new_hash = vnet_classify_hash_packet (t, key_minus_skip);
312 new_hash >>= t->log2_nbuckets;
313 new_hash &= (1 << new_log2_pages) - 1;
315 for (j = 0; j < t->entries_per_page; j++)
317 new_v = vnet_classify_entry_at_index (t, new_values,
320 if (vnet_classify_entry_is_free (new_v))
322 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
323 + (t->match_n_vectors * sizeof (u32x4)));
324 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
328 /* Crap. Tell caller to try again */
329 vnet_classify_entry_free (t, new_values, new_log2_pages);
338 static vnet_classify_entry_t *
339 split_and_rehash_linear (vnet_classify_table_t * t,
340 vnet_classify_entry_t * old_values,
341 u32 old_log2_pages, u32 new_log2_pages)
343 vnet_classify_entry_t *new_values, *v, *new_v;
344 int i, j, new_length_in_entries, old_length_in_entries;
346 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
347 new_length_in_entries = (1 << new_log2_pages) * t->entries_per_page;
348 old_length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
351 for (i = 0; i < old_length_in_entries; i++)
353 v = vnet_classify_entry_at_index (t, old_values, i);
355 if (vnet_classify_entry_is_busy (v))
357 for (; j < new_length_in_entries; j++)
359 new_v = vnet_classify_entry_at_index (t, new_values, j);
361 if (vnet_classify_entry_is_busy (new_v))
363 clib_warning ("BUG: linear rehash new entry not free!");
366 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
367 + (t->match_n_vectors * sizeof (u32x4)));
368 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
373 * Crap. Tell caller to try again.
374 * This should never happen...
376 clib_warning ("BUG: linear rehash failed!");
377 vnet_classify_entry_free (t, new_values, new_log2_pages);
388 vnet_classify_entry_claim_resource (vnet_classify_entry_t * e)
392 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
393 fib_table_lock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
395 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
396 fib_table_lock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
398 case CLASSIFY_ACTION_SET_METADATA:
404 vnet_classify_entry_release_resource (vnet_classify_entry_t * e)
408 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
409 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
411 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
412 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
414 case CLASSIFY_ACTION_SET_METADATA:
420 vnet_classify_add_del (vnet_classify_table_t * t,
421 vnet_classify_entry_t * add_v, int is_add)
424 vnet_classify_bucket_t *b, tmp_b;
425 vnet_classify_entry_t *v, *new_v, *save_new_v, *working_copy, *save_v;
431 u32 old_log2_pages, new_log2_pages;
432 u32 thread_index = vlib_get_thread_index ();
434 int resplit_once = 0;
435 int mark_bucket_linear;
437 ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
439 key_minus_skip = (u8 *) add_v->key;
440 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
442 hash = vnet_classify_hash_packet (t, key_minus_skip);
444 bucket_index = hash & (t->nbuckets - 1);
445 b = &t->buckets[bucket_index];
447 hash >>= t->log2_nbuckets;
449 while (clib_atomic_test_and_set (t->writer_lock))
452 /* First elt in the bucket? */
461 v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
462 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
463 t->match_n_vectors * sizeof (u32x4));
464 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
465 vnet_classify_entry_claim_resource (v);
468 tmp_b.offset = vnet_classify_get_offset (t, v);
470 b->as_u64 = tmp_b.as_u64;
471 t->active_elements++;
476 make_working_copy (t, b);
478 save_v = vnet_classify_get_entry (t, t->saved_bucket.offset);
479 value_index = hash & ((1 << t->saved_bucket.log2_pages) - 1);
480 limit = t->entries_per_page;
481 if (PREDICT_FALSE (b->linear_search))
484 limit *= (1 << b->log2_pages);
490 * For obvious (in hindsight) reasons, see if we're supposed to
491 * replace an existing key, then look for an empty slot.
494 for (i = 0; i < limit; i++)
496 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
499 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
501 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
502 t->match_n_vectors * sizeof (u32x4));
503 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
504 vnet_classify_entry_claim_resource (v);
506 CLIB_MEMORY_BARRIER ();
507 /* Restore the previous (k,v) pairs */
508 b->as_u64 = t->saved_bucket.as_u64;
512 for (i = 0; i < limit; i++)
514 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
516 if (vnet_classify_entry_is_free (v))
518 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
519 t->match_n_vectors * sizeof (u32x4));
520 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
521 vnet_classify_entry_claim_resource (v);
523 CLIB_MEMORY_BARRIER ();
524 b->as_u64 = t->saved_bucket.as_u64;
525 t->active_elements++;
529 /* no room at the inn... split case... */
533 for (i = 0; i < limit; i++)
535 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
538 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
540 vnet_classify_entry_release_resource (v);
541 clib_memset (v, 0xff, sizeof (vnet_classify_entry_t) +
542 t->match_n_vectors * sizeof (u32x4));
543 v->flags |= VNET_CLASSIFY_ENTRY_FREE;
545 CLIB_MEMORY_BARRIER ();
546 b->as_u64 = t->saved_bucket.as_u64;
547 t->active_elements--;
552 b->as_u64 = t->saved_bucket.as_u64;
556 old_log2_pages = t->saved_bucket.log2_pages;
557 new_log2_pages = old_log2_pages + 1;
558 working_copy = t->working_copies[thread_index];
560 if (t->saved_bucket.linear_search)
563 mark_bucket_linear = 0;
565 new_v = split_and_rehash (t, working_copy, old_log2_pages, new_log2_pages);
573 new_v = split_and_rehash (t, working_copy, old_log2_pages,
581 /* pinned collisions, use linear search */
582 new_v = split_and_rehash_linear (t, working_copy, old_log2_pages,
584 /* A new linear-search bucket? */
585 if (!t->saved_bucket.linear_search)
587 mark_bucket_linear = 1;
591 /* Try to add the new entry */
594 key_minus_skip = (u8 *) add_v->key;
595 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
597 new_hash = vnet_classify_hash_packet_inline (t, key_minus_skip);
598 new_hash >>= t->log2_nbuckets;
599 new_hash &= (1 << new_log2_pages) - 1;
601 limit = t->entries_per_page;
602 if (mark_bucket_linear)
604 limit *= (1 << new_log2_pages);
608 for (i = 0; i < limit; i++)
610 new_v = vnet_classify_entry_at_index (t, save_new_v, new_hash + i);
612 if (vnet_classify_entry_is_free (new_v))
614 clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) +
615 t->match_n_vectors * sizeof (u32x4));
616 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
617 vnet_classify_entry_claim_resource (new_v);
622 /* Crap. Try again */
623 vnet_classify_entry_free (t, save_new_v, new_log2_pages);
632 tmp_b.log2_pages = new_log2_pages;
633 tmp_b.offset = vnet_classify_get_offset (t, save_new_v);
634 tmp_b.linear_search = mark_bucket_linear;
636 CLIB_MEMORY_BARRIER ();
637 b->as_u64 = tmp_b.as_u64;
638 t->active_elements++;
639 v = vnet_classify_get_entry (t, t->saved_bucket.offset);
640 vnet_classify_entry_free (t, v, old_log2_pages);
643 CLIB_MEMORY_BARRIER ();
644 t->writer_lock[0] = 0;
649 typedef CLIB_PACKED(struct {
650 ethernet_header_t eh;
652 }) classify_data_or_mask_t;
656 vnet_classify_hash_packet (vnet_classify_table_t * t, u8 * h)
658 return vnet_classify_hash_packet_inline (t, h);
661 vnet_classify_entry_t *
662 vnet_classify_find_entry (vnet_classify_table_t * t,
663 u8 * h, u64 hash, f64 now)
665 return vnet_classify_find_entry_inline (t, h, hash, now);
669 format_classify_entry (u8 * s, va_list * args)
671 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
672 vnet_classify_entry_t *e = va_arg (*args, vnet_classify_entry_t *);
675 (s, "[%u]: next_index %d advance %d opaque %d action %d metadata %d\n",
676 vnet_classify_get_offset (t, e), e->next_index, e->advance,
677 e->opaque_index, e->action, e->metadata);
680 s = format (s, " k: %U\n", format_hex_bytes, e->key,
681 t->match_n_vectors * sizeof (u32x4));
683 if (vnet_classify_entry_is_busy (e))
684 s = format (s, " hits %lld, last_heard %.2f\n",
685 e->hits, e->last_heard);
687 s = format (s, " entry is free\n");
692 format_classify_table (u8 * s, va_list * args)
694 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
695 int verbose = va_arg (*args, int);
696 vnet_classify_bucket_t *b;
697 vnet_classify_entry_t *v, *save_v;
699 u64 active_elements = 0;
701 for (i = 0; i < t->nbuckets; i++)
707 s = format (s, "[%d]: empty\n", i);
713 s = format (s, "[%d]: heap offset %d, elts %d, %s\n", i,
714 b->offset, (1 << b->log2_pages) * t->entries_per_page,
715 b->linear_search ? "LINEAR" : "normal");
718 save_v = vnet_classify_get_entry (t, b->offset);
719 for (j = 0; j < (1 << b->log2_pages); j++)
721 for (k = 0; k < t->entries_per_page; k++)
724 v = vnet_classify_entry_at_index (t, save_v,
725 j * t->entries_per_page + k);
727 if (vnet_classify_entry_is_free (v))
730 s = format (s, " %d: empty\n",
731 j * t->entries_per_page + k);
736 s = format (s, " %d: %U\n",
737 j * t->entries_per_page + k,
738 format_classify_entry, t, v);
745 s = format (s, " %lld active elements\n", active_elements);
746 s = format (s, " %d free lists\n", vec_len (t->freelists));
747 s = format (s, " %d linear-search buckets\n", t->linear_buckets);
752 vnet_classify_add_del_table (vnet_classify_main_t * cm,
758 u32 next_table_index,
761 u8 current_data_flag,
762 i16 current_data_offset,
763 int is_add, int del_chain)
765 vnet_classify_table_t *t;
769 if (*table_index == ~0) /* add */
771 if (memory_size == 0)
772 return VNET_API_ERROR_INVALID_MEMORY_SIZE;
775 return VNET_API_ERROR_INVALID_VALUE;
777 t = vnet_classify_new_table (cm, mask, nbuckets, memory_size,
779 t->next_table_index = next_table_index;
780 t->miss_next_index = miss_next_index;
781 t->current_data_flag = current_data_flag;
782 t->current_data_offset = current_data_offset;
783 *table_index = t - cm->tables;
787 vnet_classify_main_t *cm = &vnet_classify_main;
788 t = pool_elt_at_index (cm->tables, *table_index);
790 t->next_table_index = next_table_index;
795 vnet_classify_delete_table_index (cm, *table_index, del_chain);
799 #define foreach_tcp_proto_field \
803 #define foreach_udp_proto_field \
807 #define foreach_ip4_proto_field \
818 unformat_tcp_mask (unformat_input_t * input, va_list * args)
820 u8 **maskp = va_arg (*args, u8 **);
822 u8 found_something = 0;
826 foreach_tcp_proto_field;
829 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
832 #define _(a) else if (unformat (input, #a)) a=1;
833 foreach_tcp_proto_field
839 #define _(a) found_something += a;
840 foreach_tcp_proto_field;
843 if (found_something == 0)
846 vec_validate (mask, sizeof (*tcp) - 1);
848 tcp = (tcp_header_t *) mask;
850 #define _(a) if (a) clib_memset (&tcp->a, 0xff, sizeof (tcp->a));
851 foreach_tcp_proto_field;
859 unformat_udp_mask (unformat_input_t * input, va_list * args)
861 u8 **maskp = va_arg (*args, u8 **);
863 u8 found_something = 0;
867 foreach_udp_proto_field;
870 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
873 #define _(a) else if (unformat (input, #a)) a=1;
874 foreach_udp_proto_field
880 #define _(a) found_something += a;
881 foreach_udp_proto_field;
884 if (found_something == 0)
887 vec_validate (mask, sizeof (*udp) - 1);
889 udp = (udp_header_t *) mask;
891 #define _(a) if (a) clib_memset (&udp->a, 0xff, sizeof (udp->a));
892 foreach_udp_proto_field;
901 u16 src_port, dst_port;
905 unformat_l4_mask (unformat_input_t * input, va_list * args)
907 u8 **maskp = va_arg (*args, u8 **);
908 u16 src_port = 0, dst_port = 0;
909 tcpudp_header_t *tcpudp;
911 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
913 if (unformat (input, "tcp %U", unformat_tcp_mask, maskp))
915 else if (unformat (input, "udp %U", unformat_udp_mask, maskp))
917 else if (unformat (input, "src_port"))
919 else if (unformat (input, "dst_port"))
925 if (!src_port && !dst_port)
929 vec_validate (mask, sizeof (tcpudp_header_t) - 1);
931 tcpudp = (tcpudp_header_t *) mask;
932 tcpudp->src_port = src_port;
933 tcpudp->dst_port = dst_port;
941 unformat_ip4_mask (unformat_input_t * input, va_list * args)
943 u8 **maskp = va_arg (*args, u8 **);
945 u8 found_something = 0;
949 foreach_ip4_proto_field;
955 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
957 if (unformat (input, "version"))
959 else if (unformat (input, "hdr_length"))
961 else if (unformat (input, "src"))
963 else if (unformat (input, "dst"))
965 else if (unformat (input, "proto"))
968 #define _(a) else if (unformat (input, #a)) a=1;
969 foreach_ip4_proto_field
975 #define _(a) found_something += a;
976 foreach_ip4_proto_field;
979 if (found_something == 0)
982 vec_validate (mask, sizeof (*ip) - 1);
984 ip = (ip4_header_t *) mask;
986 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
987 foreach_ip4_proto_field;
990 ip->ip_version_and_header_length = 0;
993 ip->ip_version_and_header_length |= 0xF0;
996 ip->ip_version_and_header_length |= 0x0F;
1002 #define foreach_ip6_proto_field \
1010 unformat_ip6_mask (unformat_input_t * input, va_list * args)
1012 u8 **maskp = va_arg (*args, u8 **);
1014 u8 found_something = 0;
1016 u32 ip_version_traffic_class_and_flow_label;
1018 #define _(a) u8 a=0;
1019 foreach_ip6_proto_field;
1022 u8 traffic_class = 0;
1025 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1027 if (unformat (input, "version"))
1029 else if (unformat (input, "traffic-class"))
1031 else if (unformat (input, "flow-label"))
1033 else if (unformat (input, "src"))
1035 else if (unformat (input, "dst"))
1037 else if (unformat (input, "proto"))
1040 #define _(a) else if (unformat (input, #a)) a=1;
1041 foreach_ip6_proto_field
1047 #define _(a) found_something += a;
1048 foreach_ip6_proto_field;
1051 if (found_something == 0)
1054 vec_validate (mask, sizeof (*ip) - 1);
1056 ip = (ip6_header_t *) mask;
1058 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
1059 foreach_ip6_proto_field;
1062 ip_version_traffic_class_and_flow_label = 0;
1065 ip_version_traffic_class_and_flow_label |= 0xF0000000;
1068 ip_version_traffic_class_and_flow_label |= 0x0FF00000;
1071 ip_version_traffic_class_and_flow_label |= 0x000FFFFF;
1073 ip->ip_version_traffic_class_and_flow_label =
1074 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1081 unformat_l3_mask (unformat_input_t * input, va_list * args)
1083 u8 **maskp = va_arg (*args, u8 **);
1085 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1087 if (unformat (input, "ip4 %U", unformat_ip4_mask, maskp))
1089 else if (unformat (input, "ip6 %U", unformat_ip6_mask, maskp))
1098 unformat_l2_mask (unformat_input_t * input, va_list * args)
1100 u8 **maskp = va_arg (*args, u8 **);
1115 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1117 if (unformat (input, "src"))
1119 else if (unformat (input, "dst"))
1121 else if (unformat (input, "proto"))
1123 else if (unformat (input, "tag1"))
1125 else if (unformat (input, "tag2"))
1127 else if (unformat (input, "ignore-tag1"))
1129 else if (unformat (input, "ignore-tag2"))
1131 else if (unformat (input, "cos1"))
1133 else if (unformat (input, "cos2"))
1135 else if (unformat (input, "dot1q"))
1137 else if (unformat (input, "dot1ad"))
1142 if ((src + dst + proto + tag1 + tag2 + dot1q + dot1ad +
1143 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
1146 if (tag1 || ignore_tag1 || cos1 || dot1q)
1148 if (tag2 || ignore_tag2 || cos2 || dot1ad)
1151 vec_validate (mask, len - 1);
1154 clib_memset (mask, 0xff, 6);
1157 clib_memset (mask + 6, 0xff, 6);
1161 /* inner vlan tag */
1170 mask[21] = mask[20] = 0xff;
1191 mask[16] = mask[17] = 0xff;
1200 mask[12] = mask[13] = 0xff;
1207 unformat_classify_mask (unformat_input_t * input, va_list * args)
1209 u8 **maskp = va_arg (*args, u8 **);
1210 u32 *skipp = va_arg (*args, u32 *);
1211 u32 *matchp = va_arg (*args, u32 *);
1219 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1221 if (unformat (input, "hex %U", unformat_hex_string, &mask))
1223 else if (unformat (input, "l2 %U", unformat_l2_mask, &l2))
1225 else if (unformat (input, "l3 %U", unformat_l3_mask, &l3))
1227 else if (unformat (input, "l4 %U", unformat_l4_mask, &l4))
1241 if (mask || l2 || l3 || l4)
1245 /* "With a free Ethernet header in every package" */
1247 vec_validate (l2, 13);
1251 vec_append (mask, l3);
1256 vec_append (mask, l4);
1261 /* Scan forward looking for the first significant mask octet */
1262 for (i = 0; i < vec_len (mask); i++)
1266 /* compute (skip, match) params */
1267 *skipp = i / sizeof (u32x4);
1268 vec_delete (mask, *skipp * sizeof (u32x4), 0);
1270 /* Pad mask to an even multiple of the vector size */
1271 while (vec_len (mask) % sizeof (u32x4))
1274 match = vec_len (mask) / sizeof (u32x4);
1276 for (i = match * sizeof (u32x4); i > 0; i -= sizeof (u32x4))
1278 u64 *tmp = (u64 *) (mask + (i - sizeof (u32x4)));
1279 if (*tmp || *(tmp + 1))
1284 clib_warning ("BUG: match 0");
1286 _vec_len (mask) = match * sizeof (u32x4);
1297 #define foreach_l2_input_next \
1299 _(ethernet, ETHERNET_INPUT) \
1305 unformat_l2_input_next_index (unformat_input_t * input, va_list * args)
1307 vnet_classify_main_t *cm = &vnet_classify_main;
1308 u32 *miss_next_indexp = va_arg (*args, u32 *);
1313 /* First try registered unformat fns, allowing override... */
1314 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1316 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1324 if (unformat (input, #n)) { next_index = L2_INPUT_CLASSIFY_NEXT_##N; goto out;}
1325 foreach_l2_input_next;
1328 if (unformat (input, "%d", &tmp))
1337 *miss_next_indexp = next_index;
1341 #define foreach_l2_output_next \
1345 unformat_l2_output_next_index (unformat_input_t * input, va_list * args)
1347 vnet_classify_main_t *cm = &vnet_classify_main;
1348 u32 *miss_next_indexp = va_arg (*args, u32 *);
1353 /* First try registered unformat fns, allowing override... */
1354 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1356 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1364 if (unformat (input, #n)) { next_index = L2_OUTPUT_CLASSIFY_NEXT_##N; goto out;}
1365 foreach_l2_output_next;
1368 if (unformat (input, "%d", &tmp))
1377 *miss_next_indexp = next_index;
1381 #define foreach_ip_next \
1386 unformat_ip_next_index (unformat_input_t * input, va_list * args)
1388 u32 *miss_next_indexp = va_arg (*args, u32 *);
1389 vnet_classify_main_t *cm = &vnet_classify_main;
1394 /* First try registered unformat fns, allowing override... */
1395 for (i = 0; i < vec_len (cm->unformat_ip_next_index_fns); i++)
1397 if (unformat (input, "%U", cm->unformat_ip_next_index_fns[i], &tmp))
1405 if (unformat (input, #n)) { next_index = IP_LOOKUP_NEXT_##N; goto out;}
1409 if (unformat (input, "%d", &tmp))
1418 *miss_next_indexp = next_index;
1422 #define foreach_acl_next \
1426 unformat_acl_next_index (unformat_input_t * input, va_list * args)
1428 u32 *next_indexp = va_arg (*args, u32 *);
1429 vnet_classify_main_t *cm = &vnet_classify_main;
1434 /* First try registered unformat fns, allowing override... */
1435 for (i = 0; i < vec_len (cm->unformat_acl_next_index_fns); i++)
1437 if (unformat (input, "%U", cm->unformat_acl_next_index_fns[i], &tmp))
1445 if (unformat (input, #n)) { next_index = ACL_NEXT_INDEX_##N; goto out;}
1449 if (unformat (input, "permit"))
1454 else if (unformat (input, "%d", &tmp))
1463 *next_indexp = next_index;
1468 unformat_policer_next_index (unformat_input_t * input, va_list * args)
1470 u32 *next_indexp = va_arg (*args, u32 *);
1471 vnet_classify_main_t *cm = &vnet_classify_main;
1476 /* First try registered unformat fns, allowing override... */
1477 for (i = 0; i < vec_len (cm->unformat_policer_next_index_fns); i++)
1480 (input, "%U", cm->unformat_policer_next_index_fns[i], &tmp))
1487 if (unformat (input, "%d", &tmp))
1496 *next_indexp = next_index;
1500 static clib_error_t *
1501 classify_table_command_fn (vlib_main_t * vm,
1502 unformat_input_t * input, vlib_cli_command_t * cmd)
1509 u32 table_index = ~0;
1510 u32 next_table_index = ~0;
1511 u32 miss_next_index = ~0;
1512 u32 memory_size = 2 << 20;
1514 u32 current_data_flag = 0;
1515 int current_data_offset = 0;
1518 vnet_classify_main_t *cm = &vnet_classify_main;
1521 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1523 if (unformat (input, "del"))
1525 else if (unformat (input, "del-chain"))
1530 else if (unformat (input, "buckets %d", &nbuckets))
1532 else if (unformat (input, "skip %d", &skip))
1534 else if (unformat (input, "match %d", &match))
1536 else if (unformat (input, "table %d", &table_index))
1538 else if (unformat (input, "mask %U", unformat_classify_mask,
1539 &mask, &skip, &match))
1541 else if (unformat (input, "memory-size %uM", &tmp))
1542 memory_size = tmp << 20;
1543 else if (unformat (input, "memory-size %uG", &tmp))
1544 memory_size = tmp << 30;
1545 else if (unformat (input, "next-table %d", &next_table_index))
1547 else if (unformat (input, "miss-next %U", unformat_ip_next_index,
1552 (input, "l2-input-miss-next %U", unformat_l2_input_next_index,
1557 (input, "l2-output-miss-next %U", unformat_l2_output_next_index,
1560 else if (unformat (input, "acl-miss-next %U", unformat_acl_next_index,
1563 else if (unformat (input, "current-data-flag %d", ¤t_data_flag))
1566 if (unformat (input, "current-data-offset %d", ¤t_data_offset))
1573 if (is_add && mask == 0 && table_index == ~0)
1574 return clib_error_return (0, "Mask required");
1576 if (is_add && skip == ~0 && table_index == ~0)
1577 return clib_error_return (0, "skip count required");
1579 if (is_add && match == ~0 && table_index == ~0)
1580 return clib_error_return (0, "match count required");
1582 if (!is_add && table_index == ~0)
1583 return clib_error_return (0, "table index required for delete");
1585 rv = vnet_classify_add_del_table (cm, mask, nbuckets, memory_size,
1586 skip, match, next_table_index,
1587 miss_next_index, &table_index,
1588 current_data_flag, current_data_offset,
1596 return clib_error_return (0, "vnet_classify_add_del_table returned %d",
1603 VLIB_CLI_COMMAND (classify_table, static) = {
1604 .path = "classify table",
1606 "classify table [miss-next|l2-miss_next|acl-miss-next <next_index>]"
1607 "\n mask <mask-value> buckets <nn> [skip <n>] [match <n>]"
1608 "\n [current-data-flag <n>] [current-data-offset <n>] [table <n>]"
1609 "\n [memory-size <nn>[M][G]] [next-table <n>]"
1610 "\n [del] [del-chain]",
1611 .function = classify_table_command_fn,
1616 format_vnet_classify_table (u8 * s, va_list * args)
1618 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
1619 int verbose = va_arg (*args, int);
1620 u32 index = va_arg (*args, u32);
1621 vnet_classify_table_t *t;
1625 s = format (s, "%10s%10s%10s%10s", "TableIdx", "Sessions", "NextTbl",
1626 "NextNode", verbose ? "Details" : "");
1630 t = pool_elt_at_index (cm->tables, index);
1631 s = format (s, "%10u%10d%10d%10d", index, t->active_elements,
1632 t->next_table_index, t->miss_next_index);
1634 s = format (s, "\n Heap: %U", format_mheap, t->mheap, 0 /*verbose */ );
1636 s = format (s, "\n nbuckets %d, skip %d match %d flag %d offset %d",
1637 t->nbuckets, t->skip_n_vectors, t->match_n_vectors,
1638 t->current_data_flag, t->current_data_offset);
1639 s = format (s, "\n mask %U", format_hex_bytes, t->mask,
1640 t->match_n_vectors * sizeof (u32x4));
1641 s = format (s, "\n linear-search buckets %d\n", t->linear_buckets);
1646 s = format (s, "\n%U", format_classify_table, t, verbose);
1651 static clib_error_t *
1652 show_classify_tables_command_fn (vlib_main_t * vm,
1653 unformat_input_t * input,
1654 vlib_cli_command_t * cmd)
1656 vnet_classify_main_t *cm = &vnet_classify_main;
1657 vnet_classify_table_t *t;
1658 u32 match_index = ~0;
1663 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1665 if (unformat (input, "index %d", &match_index))
1667 else if (unformat (input, "verbose %d", &verbose))
1669 else if (unformat (input, "verbose"))
1676 pool_foreach (t, cm->tables,
1678 if (match_index == ~0 || (match_index == t - cm->tables))
1679 vec_add1 (indices, t - cm->tables);
1683 if (vec_len (indices))
1685 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm, verbose,
1687 for (i = 0; i < vec_len (indices); i++)
1688 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm,
1689 verbose, indices[i]);
1692 vlib_cli_output (vm, "No classifier tables configured");
1700 VLIB_CLI_COMMAND (show_classify_table_command, static) = {
1701 .path = "show classify tables",
1702 .short_help = "show classify tables [index <nn>]",
1703 .function = show_classify_tables_command_fn,
1708 unformat_l4_match (unformat_input_t * input, va_list * args)
1710 u8 **matchp = va_arg (*args, u8 **);
1712 u8 *proto_header = 0;
1718 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1720 if (unformat (input, "src_port %d", &src_port))
1722 else if (unformat (input, "dst_port %d", &dst_port))
1728 h.src_port = clib_host_to_net_u16 (src_port);
1729 h.dst_port = clib_host_to_net_u16 (dst_port);
1730 vec_validate (proto_header, sizeof (h) - 1);
1731 memcpy (proto_header, &h, sizeof (h));
1733 *matchp = proto_header;
1739 unformat_ip4_match (unformat_input_t * input, va_list * args)
1741 u8 **matchp = va_arg (*args, u8 **);
1748 int src = 0, dst = 0;
1749 ip4_address_t src_val, dst_val;
1756 int fragment_id = 0;
1757 u32 fragment_id_val;
1763 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1765 if (unformat (input, "version %d", &version_val))
1767 else if (unformat (input, "hdr_length %d", &hdr_length_val))
1769 else if (unformat (input, "src %U", unformat_ip4_address, &src_val))
1771 else if (unformat (input, "dst %U", unformat_ip4_address, &dst_val))
1773 else if (unformat (input, "proto %d", &proto_val))
1775 else if (unformat (input, "tos %d", &tos_val))
1777 else if (unformat (input, "length %d", &length_val))
1779 else if (unformat (input, "fragment_id %d", &fragment_id_val))
1781 else if (unformat (input, "ttl %d", &ttl_val))
1783 else if (unformat (input, "checksum %d", &checksum_val))
1789 if (version + hdr_length + src + dst + proto + tos + length + fragment_id
1790 + ttl + checksum == 0)
1794 * Aligned because we use the real comparison functions
1796 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
1798 ip = (ip4_header_t *) match;
1800 /* These are realistically matched in practice */
1802 ip->src_address.as_u32 = src_val.as_u32;
1805 ip->dst_address.as_u32 = dst_val.as_u32;
1808 ip->protocol = proto_val;
1811 /* These are not, but they're included for completeness */
1813 ip->ip_version_and_header_length |= (version_val & 0xF) << 4;
1816 ip->ip_version_and_header_length |= (hdr_length_val & 0xF);
1822 ip->length = clib_host_to_net_u16 (length_val);
1828 ip->checksum = clib_host_to_net_u16 (checksum_val);
1835 unformat_ip6_match (unformat_input_t * input, va_list * args)
1837 u8 **matchp = va_arg (*args, u8 **);
1842 u8 traffic_class = 0;
1843 u32 traffic_class_val;
1846 int src = 0, dst = 0;
1847 ip6_address_t src_val, dst_val;
1850 int payload_length = 0;
1851 u32 payload_length_val;
1854 u32 ip_version_traffic_class_and_flow_label;
1856 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1858 if (unformat (input, "version %d", &version_val))
1860 else if (unformat (input, "traffic_class %d", &traffic_class_val))
1862 else if (unformat (input, "flow_label %d", &flow_label_val))
1864 else if (unformat (input, "src %U", unformat_ip6_address, &src_val))
1866 else if (unformat (input, "dst %U", unformat_ip6_address, &dst_val))
1868 else if (unformat (input, "proto %d", &proto_val))
1870 else if (unformat (input, "payload_length %d", &payload_length_val))
1872 else if (unformat (input, "hop_limit %d", &hop_limit_val))
1878 if (version + traffic_class + flow_label + src + dst + proto +
1879 payload_length + hop_limit == 0)
1883 * Aligned because we use the real comparison functions
1885 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
1887 ip = (ip6_header_t *) match;
1890 clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address));
1893 clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
1896 ip->protocol = proto_val;
1898 ip_version_traffic_class_and_flow_label = 0;
1901 ip_version_traffic_class_and_flow_label |= (version_val & 0xF) << 28;
1904 ip_version_traffic_class_and_flow_label |=
1905 (traffic_class_val & 0xFF) << 20;
1908 ip_version_traffic_class_and_flow_label |= (flow_label_val & 0xFFFFF);
1910 ip->ip_version_traffic_class_and_flow_label =
1911 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1914 ip->payload_length = clib_host_to_net_u16 (payload_length_val);
1917 ip->hop_limit = hop_limit_val;
1924 unformat_l3_match (unformat_input_t * input, va_list * args)
1926 u8 **matchp = va_arg (*args, u8 **);
1928 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1930 if (unformat (input, "ip4 %U", unformat_ip4_match, matchp))
1932 else if (unformat (input, "ip6 %U", unformat_ip6_match, matchp))
1942 unformat_vlan_tag (unformat_input_t * input, va_list * args)
1944 u8 *tagp = va_arg (*args, u8 *);
1947 if (unformat (input, "%d", &tag))
1949 tagp[0] = (tag >> 8) & 0x0F;
1950 tagp[1] = tag & 0xFF;
1958 unformat_l2_match (unformat_input_t * input, va_list * args)
1960 u8 **matchp = va_arg (*args, u8 **);
1980 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1982 if (unformat (input, "src %U", unformat_ethernet_address, &src_val))
1985 if (unformat (input, "dst %U", unformat_ethernet_address, &dst_val))
1987 else if (unformat (input, "proto %U",
1988 unformat_ethernet_type_host_byte_order, &proto_val))
1990 else if (unformat (input, "tag1 %U", unformat_vlan_tag, tag1_val))
1992 else if (unformat (input, "tag2 %U", unformat_vlan_tag, tag2_val))
1994 else if (unformat (input, "ignore-tag1"))
1996 else if (unformat (input, "ignore-tag2"))
1998 else if (unformat (input, "cos1 %d", &cos1_val))
2000 else if (unformat (input, "cos2 %d", &cos2_val))
2005 if ((src + dst + proto + tag1 + tag2 +
2006 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
2009 if (tag1 || ignore_tag1 || cos1)
2011 if (tag2 || ignore_tag2 || cos2)
2014 vec_validate_aligned (match, len - 1, sizeof (u32x4));
2017 clib_memcpy_fast (match, dst_val, 6);
2020 clib_memcpy_fast (match + 6, src_val, 6);
2024 /* inner vlan tag */
2025 match[19] = tag2_val[1];
2026 match[18] = tag2_val[0];
2028 match[18] |= (cos2_val & 0x7) << 5;
2031 match[21] = proto_val & 0xff;
2032 match[20] = proto_val >> 8;
2036 match[15] = tag1_val[1];
2037 match[14] = tag1_val[0];
2040 match[14] |= (cos1_val & 0x7) << 5;
2046 match[15] = tag1_val[1];
2047 match[14] = tag1_val[0];
2050 match[17] = proto_val & 0xff;
2051 match[16] = proto_val >> 8;
2054 match[14] |= (cos1_val & 0x7) << 5;
2060 match[18] |= (cos2_val & 0x7) << 5;
2062 match[14] |= (cos1_val & 0x7) << 5;
2065 match[13] = proto_val & 0xff;
2066 match[12] = proto_val >> 8;
2075 unformat_classify_match (unformat_input_t * input, va_list * args)
2077 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
2078 u8 **matchp = va_arg (*args, u8 **);
2079 u32 table_index = va_arg (*args, u32);
2080 vnet_classify_table_t *t;
2087 if (pool_is_free_index (cm->tables, table_index))
2090 t = pool_elt_at_index (cm->tables, table_index);
2092 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2094 if (unformat (input, "hex %U", unformat_hex_string, &match))
2096 else if (unformat (input, "l2 %U", unformat_l2_match, &l2))
2098 else if (unformat (input, "l3 %U", unformat_l3_match, &l3))
2100 else if (unformat (input, "l4 %U", unformat_l4_match, &l4))
2114 if (match || l2 || l3 || l4)
2118 /* "Win a free Ethernet header in every packet" */
2120 vec_validate_aligned (l2, 13, sizeof (u32x4));
2124 vec_append_aligned (match, l3, sizeof (u32x4));
2129 vec_append_aligned (match, l4, sizeof (u32x4));
2134 /* Make sure the vector is big enough even if key is all 0's */
2135 vec_validate_aligned
2137 ((t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4)) - 1,
2140 /* Set size, include skipped vectors */
2142 (t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4);
2153 vnet_classify_add_del_session (vnet_classify_main_t * cm,
2159 u8 action, u32 metadata, int is_add)
2161 vnet_classify_table_t *t;
2162 vnet_classify_entry_5_t _max_e __attribute__ ((aligned (16)));
2163 vnet_classify_entry_t *e;
2166 if (pool_is_free_index (cm->tables, table_index))
2167 return VNET_API_ERROR_NO_SUCH_TABLE;
2169 t = pool_elt_at_index (cm->tables, table_index);
2171 e = (vnet_classify_entry_t *) & _max_e;
2172 e->next_index = hit_next_index;
2173 e->opaque_index = opaque_index;
2174 e->advance = advance;
2179 if (e->action == CLASSIFY_ACTION_SET_IP4_FIB_INDEX)
2180 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2182 FIB_SOURCE_CLASSIFY);
2183 else if (e->action == CLASSIFY_ACTION_SET_IP6_FIB_INDEX)
2184 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2186 FIB_SOURCE_CLASSIFY);
2187 else if (e->action == CLASSIFY_ACTION_SET_METADATA)
2188 e->metadata = metadata;
2192 /* Copy key data, honoring skip_n_vectors */
2193 clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
2194 t->match_n_vectors * sizeof (u32x4));
2196 /* Clear don't-care bits; likely when dynamically creating sessions */
2197 for (i = 0; i < t->match_n_vectors; i++)
2198 e->key[i] &= t->mask[i];
2200 rv = vnet_classify_add_del (t, e, is_add);
2202 vnet_classify_entry_release_resource (e);
2205 return VNET_API_ERROR_NO_SUCH_ENTRY;
2209 static clib_error_t *
2210 classify_session_command_fn (vlib_main_t * vm,
2211 unformat_input_t * input,
2212 vlib_cli_command_t * cmd)
2214 vnet_classify_main_t *cm = &vnet_classify_main;
2216 u32 table_index = ~0;
2217 u32 hit_next_index = ~0;
2218 u64 opaque_index = ~0;
2225 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2227 if (unformat (input, "del"))
2229 else if (unformat (input, "hit-next %U", unformat_ip_next_index,
2234 (input, "l2-input-hit-next %U", unformat_l2_input_next_index,
2239 (input, "l2-output-hit-next %U", unformat_l2_output_next_index,
2242 else if (unformat (input, "acl-hit-next %U", unformat_acl_next_index,
2245 else if (unformat (input, "policer-hit-next %U",
2246 unformat_policer_next_index, &hit_next_index))
2248 else if (unformat (input, "opaque-index %lld", &opaque_index))
2250 else if (unformat (input, "match %U", unformat_classify_match,
2251 cm, &match, table_index))
2253 else if (unformat (input, "advance %d", &advance))
2255 else if (unformat (input, "table-index %d", &table_index))
2257 else if (unformat (input, "action set-ip4-fib-id %d", &metadata))
2259 else if (unformat (input, "action set-ip6-fib-id %d", &metadata))
2261 else if (unformat (input, "action set-sr-policy-index %d", &metadata))
2265 /* Try registered opaque-index unformat fns */
2266 for (i = 0; i < vec_len (cm->unformat_opaque_index_fns); i++)
2268 if (unformat (input, "%U", cm->unformat_opaque_index_fns[i],
2278 if (table_index == ~0)
2279 return clib_error_return (0, "Table index required");
2281 if (is_add && match == 0)
2282 return clib_error_return (0, "Match value required");
2284 rv = vnet_classify_add_del_session (cm, table_index, match,
2286 opaque_index, advance,
2287 action, metadata, is_add);
2295 return clib_error_return (0,
2296 "vnet_classify_add_del_session returned %d",
2304 VLIB_CLI_COMMAND (classify_session_command, static) = {
2305 .path = "classify session",
2307 "classify session [hit-next|l2-input-hit-next|l2-output-hit-next|"
2308 "acl-hit-next <next_index>|policer-hit-next <policer_name>]"
2309 "\n table-index <nn> match [hex] [l2] [l3 ip4] [opaque-index <index>]"
2310 "\n [action set-ip4-fib-id|set-ip6-fib-id|set-sr-policy-index <n>] [del]",
2311 .function = classify_session_command_fn,
2316 unformat_opaque_sw_if_index (unformat_input_t * input, va_list * args)
2318 u64 *opaquep = va_arg (*args, u64 *);
2321 if (unformat (input, "opaque-sw_if_index %U", unformat_vnet_sw_interface,
2322 vnet_get_main (), &sw_if_index))
2324 *opaquep = sw_if_index;
2331 unformat_ip_next_node (unformat_input_t * input, va_list * args)
2333 vnet_classify_main_t *cm = &vnet_classify_main;
2334 u32 *next_indexp = va_arg (*args, u32 *);
2336 u32 next_index = ~0;
2338 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2339 cm->vlib_main, &node_index))
2341 next_index = vlib_node_add_next (cm->vlib_main,
2342 ip6_classify_node.index, node_index);
2344 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2345 cm->vlib_main, &node_index))
2347 next_index = vlib_node_add_next (cm->vlib_main,
2348 ip4_classify_node.index, node_index);
2353 *next_indexp = next_index;
2358 unformat_acl_next_node (unformat_input_t * input, va_list * args)
2360 vnet_classify_main_t *cm = &vnet_classify_main;
2361 u32 *next_indexp = va_arg (*args, u32 *);
2365 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2366 cm->vlib_main, &node_index))
2368 next_index = vlib_node_add_next (cm->vlib_main,
2369 ip6_inacl_node.index, node_index);
2371 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2372 cm->vlib_main, &node_index))
2374 next_index = vlib_node_add_next (cm->vlib_main,
2375 ip4_inacl_node.index, node_index);
2380 *next_indexp = next_index;
2385 unformat_l2_input_next_node (unformat_input_t * input, va_list * args)
2387 vnet_classify_main_t *cm = &vnet_classify_main;
2388 u32 *next_indexp = va_arg (*args, u32 *);
2392 if (unformat (input, "input-node %U", unformat_vlib_node,
2393 cm->vlib_main, &node_index))
2395 next_index = vlib_node_add_next
2396 (cm->vlib_main, l2_input_classify_node.index, node_index);
2398 *next_indexp = next_index;
2405 unformat_l2_output_next_node (unformat_input_t * input, va_list * args)
2407 vnet_classify_main_t *cm = &vnet_classify_main;
2408 u32 *next_indexp = va_arg (*args, u32 *);
2412 if (unformat (input, "output-node %U", unformat_vlib_node,
2413 cm->vlib_main, &node_index))
2415 next_index = vlib_node_add_next
2416 (cm->vlib_main, l2_output_classify_node.index, node_index);
2418 *next_indexp = next_index;
2424 static clib_error_t *
2425 vnet_classify_init (vlib_main_t * vm)
2427 vnet_classify_main_t *cm = &vnet_classify_main;
2430 cm->vnet_main = vnet_get_main ();
2432 vnet_classify_register_unformat_opaque_index_fn
2433 (unformat_opaque_sw_if_index);
2435 vnet_classify_register_unformat_ip_next_index_fn (unformat_ip_next_node);
2437 vnet_classify_register_unformat_l2_next_index_fn
2438 (unformat_l2_input_next_node);
2440 vnet_classify_register_unformat_l2_next_index_fn
2441 (unformat_l2_output_next_node);
2443 vnet_classify_register_unformat_acl_next_index_fn (unformat_acl_next_node);
2448 VLIB_INIT_FUNCTION (vnet_classify_init);
2462 test_entry_t *entries;
2464 /* test parameters */
2470 vnet_classify_table_t *table;
2478 classify_data_or_mask_t *mask;
2479 classify_data_or_mask_t *data;
2482 vnet_classify_main_t *classify_main;
2483 vlib_main_t *vlib_main;
2485 } test_classify_main_t;
2487 static test_classify_main_t test_classify_main;
2489 static clib_error_t *
2490 test_classify_churn (test_classify_main_t * tm)
2492 classify_data_or_mask_t *mask, *data;
2493 vlib_main_t *vm = tm->vlib_main;
2495 u8 *mp = 0, *dp = 0;
2499 vec_validate_aligned (mp, 3 * sizeof (u32x4), sizeof (u32x4));
2500 vec_validate_aligned (dp, 3 * sizeof (u32x4), sizeof (u32x4));
2502 mask = (classify_data_or_mask_t *) mp;
2503 data = (classify_data_or_mask_t *) dp;
2505 /* Mask on src address */
2506 clib_memset (&mask->ip.src_address, 0xff, 4);
2508 tmp = clib_host_to_net_u32 (tm->src.as_u32);
2510 for (i = 0; i < tm->sessions; i++)
2512 vec_add2 (tm->entries, ep, 1);
2513 ep->addr.as_u32 = clib_host_to_net_u32 (tmp);
2518 tm->table = vnet_classify_new_table (tm->classify_main,
2521 tm->memory_size, 0 /* skip */ ,
2522 3 /* vectors to match */ );
2523 tm->table->miss_next_index = IP_LOOKUP_NEXT_DROP;
2524 tm->table_index = tm->table - tm->classify_main->tables;
2525 vlib_cli_output (vm, "Created table %d, buckets %d",
2526 tm->table_index, tm->buckets);
2528 vlib_cli_output (vm, "Initialize: add %d (approx. half of %d sessions)...",
2529 tm->sessions / 2, tm->sessions);
2531 for (i = 0; i < tm->sessions / 2; i++)
2533 ep = vec_elt_at_index (tm->entries, i);
2535 data->ip.src_address.as_u32 = ep->addr.as_u32;
2538 rv = vnet_classify_add_del_session (tm->classify_main,
2541 IP_LOOKUP_NEXT_DROP,
2542 i /* opaque_index */ ,
2549 clib_warning ("add: returned %d", rv);
2552 vlib_cli_output (vm, "add: %U", format_ip4_address, &ep->addr.as_u32);
2555 vlib_cli_output (vm, "Execute %d random add/delete operations",
2558 for (i = 0; i < tm->iterations; i++)
2562 /* Pick a random entry */
2563 index = random_u32 (&tm->seed) % tm->sessions;
2565 ep = vec_elt_at_index (tm->entries, index);
2567 data->ip.src_address.as_u32 = ep->addr.as_u32;
2569 /* If it's in the table, remove it. Else, add it */
2570 is_add = !ep->in_table;
2573 vlib_cli_output (vm, "%s: %U",
2574 is_add ? "add" : "del",
2575 format_ip4_address, &ep->addr.as_u32);
2577 rv = vnet_classify_add_del_session (tm->classify_main,
2580 IP_LOOKUP_NEXT_DROP,
2581 i /* opaque_index */ ,
2587 vlib_cli_output (vm,
2588 "%s[%d]: %U returned %d", is_add ? "add" : "del",
2589 index, format_ip4_address, &ep->addr.as_u32, rv);
2591 ep->in_table = is_add;
2594 vlib_cli_output (vm, "Remove remaining %d entries from the table",
2595 tm->table->active_elements);
2597 for (i = 0; i < tm->sessions; i++)
2601 vnet_classify_entry_t *e;
2603 ep = tm->entries + i;
2604 if (ep->in_table == 0)
2607 data->ip.src_address.as_u32 = ep->addr.as_u32;
2609 hash = vnet_classify_hash_packet (tm->table, (u8 *) data);
2611 e = vnet_classify_find_entry (tm->table,
2612 (u8 *) data, hash, 0 /* time_now */ );
2615 clib_warning ("Couldn't find %U index %d which should be present",
2616 format_ip4_address, ep->addr, i);
2620 key_minus_skip = (u8 *) e->key;
2621 key_minus_skip -= tm->table->skip_n_vectors * sizeof (u32x4);
2623 rv = vnet_classify_add_del_session
2626 key_minus_skip, IP_LOOKUP_NEXT_DROP, i /* opaque_index */ ,
2627 0 /* advance */ , 0, 0,
2631 clib_warning ("del: returned %d", rv);
2634 vlib_cli_output (vm, "del: %U", format_ip4_address, &ep->addr.as_u32);
2637 vlib_cli_output (vm, "%d entries remain, MUST be zero",
2638 tm->table->active_elements);
2640 vlib_cli_output (vm, "Table after cleanup: \n%U\n",
2641 format_classify_table, tm->table, 0 /* verbose */ );
2646 vnet_classify_delete_table_index (tm->classify_main,
2647 tm->table_index, 1 /* del_chain */ );
2649 tm->table_index = ~0;
2650 vec_free (tm->entries);
2655 static clib_error_t *
2656 test_classify_command_fn (vlib_main_t * vm,
2657 unformat_input_t * input, vlib_cli_command_t * cmd)
2659 test_classify_main_t *tm = &test_classify_main;
2660 vnet_classify_main_t *cm = &vnet_classify_main;
2663 clib_error_t *error = 0;
2666 tm->sessions = 8192;
2667 tm->iterations = 8192;
2668 tm->memory_size = 64 << 20;
2669 tm->src.as_u32 = clib_net_to_host_u32 (0x0100000A);
2671 tm->seed = 0xDEADDABE;
2672 tm->classify_main = cm;
2676 /* Default starting address 1.0.0.10 */
2678 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2680 if (unformat (input, "sessions %d", &tmp))
2683 if (unformat (input, "src %U", unformat_ip4_address, &tm->src.as_u32))
2685 else if (unformat (input, "buckets %d", &tm->buckets))
2687 else if (unformat (input, "memory-size %uM", &tmp))
2688 tm->memory_size = tmp << 20;
2689 else if (unformat (input, "memory-size %uG", &tmp))
2690 tm->memory_size = tmp << 30;
2691 else if (unformat (input, "seed %d", &tm->seed))
2693 else if (unformat (input, "verbose"))
2696 else if (unformat (input, "iterations %d", &tm->iterations))
2698 else if (unformat (input, "churn-test"))
2707 error = test_classify_churn (tm);
2710 error = clib_error_return (0, "No such test");
2718 VLIB_CLI_COMMAND (test_classify_command, static) = {
2719 .path = "test classify",
2721 "test classify [src <ip>] [sessions <nn>] [buckets <nn>] [seed <nnn>]\n"
2722 " [memory-size <nn>[M|G]]\n"
2724 .function = test_classify_command_fn,
2727 #endif /* TEST_CODE */
2730 * fd.io coding-style-patch-verification: ON
2733 * eval: (c-set-style "gnu")