2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/classify/vnet_classify.h>
16 #include <vnet/classify/in_out_acl.h>
17 #include <vnet/ip/ip.h>
18 #include <vnet/api_errno.h> /* for API error numbers */
19 #include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
20 #include <vnet/fib/fib_table.h>
22 vnet_classify_main_t vnet_classify_main;
24 #if VALIDATION_SCAFFOLDING
25 /* Validation scaffolding */
27 mv (vnet_classify_table_t * t)
31 oldheap = clib_mem_set_heap (t->mheap);
33 clib_mem_set_heap (oldheap);
37 rogue (vnet_classify_table_t * t)
40 vnet_classify_entry_t *v, *save_v;
41 u32 active_elements = 0;
42 vnet_classify_bucket_t *b;
44 for (i = 0; i < t->nbuckets; i++)
49 save_v = vnet_classify_get_entry (t, b->offset);
50 for (j = 0; j < (1 << b->log2_pages); j++)
52 for (k = 0; k < t->entries_per_page; k++)
54 v = vnet_classify_entry_at_index
55 (t, save_v, j * t->entries_per_page + k);
57 if (vnet_classify_entry_is_busy (v))
63 if (active_elements != t->active_elements)
64 clib_warning ("found %u expected %u elts", active_elements,
69 mv (vnet_classify_table_t * t)
74 rogue (vnet_classify_table_t * t)
80 vnet_classify_register_unformat_l2_next_index_fn (unformat_function_t * fn)
82 vnet_classify_main_t *cm = &vnet_classify_main;
84 vec_add1 (cm->unformat_l2_next_index_fns, fn);
88 vnet_classify_register_unformat_ip_next_index_fn (unformat_function_t * fn)
90 vnet_classify_main_t *cm = &vnet_classify_main;
92 vec_add1 (cm->unformat_ip_next_index_fns, fn);
96 vnet_classify_register_unformat_acl_next_index_fn (unformat_function_t * fn)
98 vnet_classify_main_t *cm = &vnet_classify_main;
100 vec_add1 (cm->unformat_acl_next_index_fns, fn);
104 vnet_classify_register_unformat_policer_next_index_fn (unformat_function_t *
107 vnet_classify_main_t *cm = &vnet_classify_main;
109 vec_add1 (cm->unformat_policer_next_index_fns, fn);
113 vnet_classify_register_unformat_opaque_index_fn (unformat_function_t * fn)
115 vnet_classify_main_t *cm = &vnet_classify_main;
117 vec_add1 (cm->unformat_opaque_index_fns, fn);
120 vnet_classify_table_t *
121 vnet_classify_new_table (vnet_classify_main_t * cm,
122 u8 * mask, u32 nbuckets, u32 memory_size,
123 u32 skip_n_vectors, u32 match_n_vectors)
125 vnet_classify_table_t *t;
128 nbuckets = 1 << (max_log2 (nbuckets));
130 pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
131 clib_memset (t, 0, sizeof (*t));
133 vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
134 clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4));
136 t->next_table_index = ~0;
137 t->nbuckets = nbuckets;
138 t->log2_nbuckets = max_log2 (nbuckets);
139 t->match_n_vectors = match_n_vectors;
140 t->skip_n_vectors = skip_n_vectors;
141 t->entries_per_page = 2;
143 #if USE_DLMALLOC == 0
144 t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
146 t->mheap = create_mspace (memory_size, 1 /* locked */ );
147 /* classifier requires the memory to be contiguous, so can not expand. */
148 mspace_disable_expand (t->mheap);
151 vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
152 oldheap = clib_mem_set_heap (t->mheap);
154 t->writer_lock = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
155 CLIB_CACHE_LINE_BYTES);
156 t->writer_lock[0] = 0;
158 clib_mem_set_heap (oldheap);
163 vnet_classify_delete_table_index (vnet_classify_main_t * cm,
164 u32 table_index, int del_chain)
166 vnet_classify_table_t *t;
168 /* Tolerate multiple frees, up to a point */
169 if (pool_is_free_index (cm->tables, table_index))
172 t = pool_elt_at_index (cm->tables, table_index);
173 if (del_chain && t->next_table_index != ~0)
174 /* Recursively delete the entire chain */
175 vnet_classify_delete_table_index (cm, t->next_table_index, del_chain);
178 vec_free (t->buckets);
179 #if USE_DLMALLOC == 0
180 mheap_free (t->mheap);
182 destroy_mspace (t->mheap);
185 pool_put (cm->tables, t);
188 static vnet_classify_entry_t *
189 vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
191 vnet_classify_entry_t *rv = 0;
195 ASSERT (t->writer_lock[0]);
197 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
198 * t->entries_per_page * (1 << log2_pages);
200 if (log2_pages >= vec_len (t->freelists) || t->freelists[log2_pages] == 0)
202 oldheap = clib_mem_set_heap (t->mheap);
204 vec_validate (t->freelists, log2_pages);
206 rv = clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
207 clib_mem_set_heap (oldheap);
210 rv = t->freelists[log2_pages];
211 t->freelists[log2_pages] = rv->next_free;
216 clib_memset (rv, 0xff, required_length);
221 vnet_classify_entry_free (vnet_classify_table_t * t,
222 vnet_classify_entry_t * v, u32 log2_pages)
224 ASSERT (t->writer_lock[0]);
226 ASSERT (vec_len (t->freelists) > log2_pages);
228 v->next_free = t->freelists[log2_pages];
229 t->freelists[log2_pages] = v;
232 static inline void make_working_copy
233 (vnet_classify_table_t * t, vnet_classify_bucket_t * b)
235 vnet_classify_entry_t *v;
236 vnet_classify_bucket_t working_bucket __attribute__ ((aligned (8)));
238 vnet_classify_entry_t *working_copy;
239 u32 thread_index = vlib_get_thread_index ();
240 int working_copy_length, required_length;
242 if (thread_index >= vec_len (t->working_copies))
244 oldheap = clib_mem_set_heap (t->mheap);
245 vec_validate (t->working_copies, thread_index);
246 vec_validate (t->working_copy_lengths, thread_index);
247 t->working_copy_lengths[thread_index] = -1;
248 clib_mem_set_heap (oldheap);
252 * working_copies are per-cpu so that near-simultaneous
253 * updates from multiple threads will not result in sporadic, spurious
256 working_copy = t->working_copies[thread_index];
257 working_copy_length = t->working_copy_lengths[thread_index];
259 (sizeof (vnet_classify_entry_t) + (t->match_n_vectors * sizeof (u32x4)))
260 * t->entries_per_page * (1 << b->log2_pages);
262 t->saved_bucket.as_u64 = b->as_u64;
263 oldheap = clib_mem_set_heap (t->mheap);
265 if (required_length > working_copy_length)
268 clib_mem_free (working_copy);
270 clib_mem_alloc_aligned (required_length, CLIB_CACHE_LINE_BYTES);
271 t->working_copies[thread_index] = working_copy;
274 clib_mem_set_heap (oldheap);
276 v = vnet_classify_get_entry (t, b->offset);
278 clib_memcpy_fast (working_copy, v, required_length);
280 working_bucket.as_u64 = b->as_u64;
281 working_bucket.offset = vnet_classify_get_offset (t, working_copy);
282 CLIB_MEMORY_BARRIER ();
283 b->as_u64 = working_bucket.as_u64;
284 t->working_copies[thread_index] = working_copy;
287 static vnet_classify_entry_t *
288 split_and_rehash (vnet_classify_table_t * t,
289 vnet_classify_entry_t * old_values, u32 old_log2_pages,
292 vnet_classify_entry_t *new_values, *v, *new_v;
293 int i, j, length_in_entries;
295 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
296 length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
298 for (i = 0; i < length_in_entries; i++)
302 v = vnet_classify_entry_at_index (t, old_values, i);
304 if (vnet_classify_entry_is_busy (v))
306 /* Hack so we can use the packet hash routine */
308 key_minus_skip = (u8 *) v->key;
309 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
311 new_hash = vnet_classify_hash_packet (t, key_minus_skip);
312 new_hash >>= t->log2_nbuckets;
313 new_hash &= (1 << new_log2_pages) - 1;
315 for (j = 0; j < t->entries_per_page; j++)
317 new_v = vnet_classify_entry_at_index (t, new_values,
320 if (vnet_classify_entry_is_free (new_v))
322 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
323 + (t->match_n_vectors * sizeof (u32x4)));
324 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
328 /* Crap. Tell caller to try again */
329 vnet_classify_entry_free (t, new_values, new_log2_pages);
338 static vnet_classify_entry_t *
339 split_and_rehash_linear (vnet_classify_table_t * t,
340 vnet_classify_entry_t * old_values,
341 u32 old_log2_pages, u32 new_log2_pages)
343 vnet_classify_entry_t *new_values, *v, *new_v;
344 int i, j, new_length_in_entries, old_length_in_entries;
346 new_values = vnet_classify_entry_alloc (t, new_log2_pages);
347 new_length_in_entries = (1 << new_log2_pages) * t->entries_per_page;
348 old_length_in_entries = (1 << old_log2_pages) * t->entries_per_page;
351 for (i = 0; i < old_length_in_entries; i++)
353 v = vnet_classify_entry_at_index (t, old_values, i);
355 if (vnet_classify_entry_is_busy (v))
357 for (; j < new_length_in_entries; j++)
359 new_v = vnet_classify_entry_at_index (t, new_values, j);
361 if (vnet_classify_entry_is_busy (new_v))
363 clib_warning ("BUG: linear rehash new entry not free!");
366 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
367 + (t->match_n_vectors * sizeof (u32x4)));
368 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
373 * Crap. Tell caller to try again.
374 * This should never happen...
376 clib_warning ("BUG: linear rehash failed!");
377 vnet_classify_entry_free (t, new_values, new_log2_pages);
388 vnet_classify_entry_claim_resource (vnet_classify_entry_t * e)
392 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
393 fib_table_lock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
395 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
396 fib_table_lock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
398 case CLASSIFY_ACTION_SET_METADATA:
404 vnet_classify_entry_release_resource (vnet_classify_entry_t * e)
408 case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
409 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
411 case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
412 fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
414 case CLASSIFY_ACTION_SET_METADATA:
420 vnet_classify_add_del (vnet_classify_table_t * t,
421 vnet_classify_entry_t * add_v, int is_add)
424 vnet_classify_bucket_t *b, tmp_b;
425 vnet_classify_entry_t *v, *new_v, *save_new_v, *working_copy, *save_v;
431 u32 old_log2_pages, new_log2_pages;
432 u32 thread_index = vlib_get_thread_index ();
434 int resplit_once = 0;
435 int mark_bucket_linear;
437 ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0);
439 key_minus_skip = (u8 *) add_v->key;
440 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
442 hash = vnet_classify_hash_packet (t, key_minus_skip);
444 bucket_index = hash & (t->nbuckets - 1);
445 b = &t->buckets[bucket_index];
447 hash >>= t->log2_nbuckets;
449 while (clib_atomic_test_and_set (t->writer_lock))
452 /* First elt in the bucket? */
461 v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
462 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
463 t->match_n_vectors * sizeof (u32x4));
464 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
465 vnet_classify_entry_claim_resource (v);
468 tmp_b.offset = vnet_classify_get_offset (t, v);
470 b->as_u64 = tmp_b.as_u64;
471 t->active_elements++;
476 make_working_copy (t, b);
478 save_v = vnet_classify_get_entry (t, t->saved_bucket.offset);
479 value_index = hash & ((1 << t->saved_bucket.log2_pages) - 1);
480 limit = t->entries_per_page;
481 if (PREDICT_FALSE (b->linear_search))
484 limit *= (1 << b->log2_pages);
490 * For obvious (in hindsight) reasons, see if we're supposed to
491 * replace an existing key, then look for an empty slot.
494 for (i = 0; i < limit; i++)
496 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
499 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
501 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
502 t->match_n_vectors * sizeof (u32x4));
503 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
504 vnet_classify_entry_claim_resource (v);
506 CLIB_MEMORY_BARRIER ();
507 /* Restore the previous (k,v) pairs */
508 b->as_u64 = t->saved_bucket.as_u64;
512 for (i = 0; i < limit; i++)
514 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
516 if (vnet_classify_entry_is_free (v))
518 clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
519 t->match_n_vectors * sizeof (u32x4));
520 v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
521 vnet_classify_entry_claim_resource (v);
523 CLIB_MEMORY_BARRIER ();
524 b->as_u64 = t->saved_bucket.as_u64;
525 t->active_elements++;
529 /* no room at the inn... split case... */
533 for (i = 0; i < limit; i++)
535 v = vnet_classify_entry_at_index (t, save_v, value_index + i);
538 (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
540 vnet_classify_entry_release_resource (v);
541 clib_memset (v, 0xff, sizeof (vnet_classify_entry_t) +
542 t->match_n_vectors * sizeof (u32x4));
543 v->flags |= VNET_CLASSIFY_ENTRY_FREE;
545 CLIB_MEMORY_BARRIER ();
546 b->as_u64 = t->saved_bucket.as_u64;
547 t->active_elements--;
552 b->as_u64 = t->saved_bucket.as_u64;
556 old_log2_pages = t->saved_bucket.log2_pages;
557 new_log2_pages = old_log2_pages + 1;
558 working_copy = t->working_copies[thread_index];
560 if (t->saved_bucket.linear_search)
563 mark_bucket_linear = 0;
565 new_v = split_and_rehash (t, working_copy, old_log2_pages, new_log2_pages);
573 new_v = split_and_rehash (t, working_copy, old_log2_pages,
581 /* pinned collisions, use linear search */
582 new_v = split_and_rehash_linear (t, working_copy, old_log2_pages,
584 /* A new linear-search bucket? */
585 if (!t->saved_bucket.linear_search)
587 mark_bucket_linear = 1;
591 /* Try to add the new entry */
594 key_minus_skip = (u8 *) add_v->key;
595 key_minus_skip -= t->skip_n_vectors * sizeof (u32x4);
597 new_hash = vnet_classify_hash_packet_inline (t, key_minus_skip);
598 new_hash >>= t->log2_nbuckets;
599 new_hash &= (1 << new_log2_pages) - 1;
601 limit = t->entries_per_page;
602 if (mark_bucket_linear)
604 limit *= (1 << new_log2_pages);
608 for (i = 0; i < limit; i++)
610 new_v = vnet_classify_entry_at_index (t, save_new_v, new_hash + i);
612 if (vnet_classify_entry_is_free (new_v))
614 clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) +
615 t->match_n_vectors * sizeof (u32x4));
616 new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
617 vnet_classify_entry_claim_resource (new_v);
622 /* Crap. Try again */
623 vnet_classify_entry_free (t, save_new_v, new_log2_pages);
631 tmp_b.log2_pages = new_log2_pages;
632 tmp_b.offset = vnet_classify_get_offset (t, save_new_v);
633 tmp_b.linear_search = mark_bucket_linear;
635 CLIB_MEMORY_BARRIER ();
636 b->as_u64 = tmp_b.as_u64;
637 t->active_elements++;
638 v = vnet_classify_get_entry (t, t->saved_bucket.offset);
639 vnet_classify_entry_free (t, v, old_log2_pages);
642 CLIB_MEMORY_BARRIER ();
643 t->writer_lock[0] = 0;
648 typedef CLIB_PACKED(struct {
649 ethernet_header_t eh;
651 }) classify_data_or_mask_t;
655 vnet_classify_hash_packet (vnet_classify_table_t * t, u8 * h)
657 return vnet_classify_hash_packet_inline (t, h);
660 vnet_classify_entry_t *
661 vnet_classify_find_entry (vnet_classify_table_t * t,
662 u8 * h, u64 hash, f64 now)
664 return vnet_classify_find_entry_inline (t, h, hash, now);
668 format_classify_entry (u8 * s, va_list * args)
670 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
671 vnet_classify_entry_t *e = va_arg (*args, vnet_classify_entry_t *);
674 (s, "[%u]: next_index %d advance %d opaque %d action %d metadata %d\n",
675 vnet_classify_get_offset (t, e), e->next_index, e->advance,
676 e->opaque_index, e->action, e->metadata);
679 s = format (s, " k: %U\n", format_hex_bytes, e->key,
680 t->match_n_vectors * sizeof (u32x4));
682 if (vnet_classify_entry_is_busy (e))
683 s = format (s, " hits %lld, last_heard %.2f\n",
684 e->hits, e->last_heard);
686 s = format (s, " entry is free\n");
691 format_classify_table (u8 * s, va_list * args)
693 vnet_classify_table_t *t = va_arg (*args, vnet_classify_table_t *);
694 int verbose = va_arg (*args, int);
695 vnet_classify_bucket_t *b;
696 vnet_classify_entry_t *v, *save_v;
698 u64 active_elements = 0;
700 for (i = 0; i < t->nbuckets; i++)
706 s = format (s, "[%d]: empty\n", i);
712 s = format (s, "[%d]: heap offset %d, elts %d, %s\n", i,
713 b->offset, (1 << b->log2_pages) * t->entries_per_page,
714 b->linear_search ? "LINEAR" : "normal");
717 save_v = vnet_classify_get_entry (t, b->offset);
718 for (j = 0; j < (1 << b->log2_pages); j++)
720 for (k = 0; k < t->entries_per_page; k++)
723 v = vnet_classify_entry_at_index (t, save_v,
724 j * t->entries_per_page + k);
726 if (vnet_classify_entry_is_free (v))
729 s = format (s, " %d: empty\n",
730 j * t->entries_per_page + k);
735 s = format (s, " %d: %U\n",
736 j * t->entries_per_page + k,
737 format_classify_entry, t, v);
744 s = format (s, " %lld active elements\n", active_elements);
745 s = format (s, " %d free lists\n", vec_len (t->freelists));
746 s = format (s, " %d linear-search buckets\n", t->linear_buckets);
751 vnet_classify_add_del_table (vnet_classify_main_t * cm,
757 u32 next_table_index,
760 u8 current_data_flag,
761 i16 current_data_offset,
762 int is_add, int del_chain)
764 vnet_classify_table_t *t;
768 if (*table_index == ~0) /* add */
770 if (memory_size == 0)
771 return VNET_API_ERROR_INVALID_MEMORY_SIZE;
774 return VNET_API_ERROR_INVALID_VALUE;
776 t = vnet_classify_new_table (cm, mask, nbuckets, memory_size,
778 t->next_table_index = next_table_index;
779 t->miss_next_index = miss_next_index;
780 t->current_data_flag = current_data_flag;
781 t->current_data_offset = current_data_offset;
782 *table_index = t - cm->tables;
786 vnet_classify_main_t *cm = &vnet_classify_main;
787 t = pool_elt_at_index (cm->tables, *table_index);
789 t->next_table_index = next_table_index;
794 vnet_classify_delete_table_index (cm, *table_index, del_chain);
798 #define foreach_tcp_proto_field \
802 #define foreach_udp_proto_field \
806 #define foreach_ip4_proto_field \
817 unformat_tcp_mask (unformat_input_t * input, va_list * args)
819 u8 **maskp = va_arg (*args, u8 **);
821 u8 found_something = 0;
825 foreach_tcp_proto_field;
828 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
831 #define _(a) else if (unformat (input, #a)) a=1;
832 foreach_tcp_proto_field
838 #define _(a) found_something += a;
839 foreach_tcp_proto_field;
842 if (found_something == 0)
845 vec_validate (mask, sizeof (*tcp) - 1);
847 tcp = (tcp_header_t *) mask;
849 #define _(a) if (a) clib_memset (&tcp->a, 0xff, sizeof (tcp->a));
850 foreach_tcp_proto_field;
858 unformat_udp_mask (unformat_input_t * input, va_list * args)
860 u8 **maskp = va_arg (*args, u8 **);
862 u8 found_something = 0;
866 foreach_udp_proto_field;
869 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
872 #define _(a) else if (unformat (input, #a)) a=1;
873 foreach_udp_proto_field
879 #define _(a) found_something += a;
880 foreach_udp_proto_field;
883 if (found_something == 0)
886 vec_validate (mask, sizeof (*udp) - 1);
888 udp = (udp_header_t *) mask;
890 #define _(a) if (a) clib_memset (&udp->a, 0xff, sizeof (udp->a));
891 foreach_udp_proto_field;
900 u16 src_port, dst_port;
904 unformat_l4_mask (unformat_input_t * input, va_list * args)
906 u8 **maskp = va_arg (*args, u8 **);
907 u16 src_port = 0, dst_port = 0;
908 tcpudp_header_t *tcpudp;
910 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
912 if (unformat (input, "tcp %U", unformat_tcp_mask, maskp))
914 else if (unformat (input, "udp %U", unformat_udp_mask, maskp))
916 else if (unformat (input, "src_port"))
918 else if (unformat (input, "dst_port"))
924 if (!src_port && !dst_port)
928 vec_validate (mask, sizeof (tcpudp_header_t) - 1);
930 tcpudp = (tcpudp_header_t *) mask;
931 tcpudp->src_port = src_port;
932 tcpudp->dst_port = dst_port;
940 unformat_ip4_mask (unformat_input_t * input, va_list * args)
942 u8 **maskp = va_arg (*args, u8 **);
944 u8 found_something = 0;
948 foreach_ip4_proto_field;
954 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
956 if (unformat (input, "version"))
958 else if (unformat (input, "hdr_length"))
960 else if (unformat (input, "src"))
962 else if (unformat (input, "dst"))
964 else if (unformat (input, "proto"))
967 #define _(a) else if (unformat (input, #a)) a=1;
968 foreach_ip4_proto_field
974 #define _(a) found_something += a;
975 foreach_ip4_proto_field;
978 if (found_something == 0)
981 vec_validate (mask, sizeof (*ip) - 1);
983 ip = (ip4_header_t *) mask;
985 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
986 foreach_ip4_proto_field;
989 ip->ip_version_and_header_length = 0;
992 ip->ip_version_and_header_length |= 0xF0;
995 ip->ip_version_and_header_length |= 0x0F;
1001 #define foreach_ip6_proto_field \
1009 unformat_ip6_mask (unformat_input_t * input, va_list * args)
1011 u8 **maskp = va_arg (*args, u8 **);
1013 u8 found_something = 0;
1015 u32 ip_version_traffic_class_and_flow_label;
1017 #define _(a) u8 a=0;
1018 foreach_ip6_proto_field;
1021 u8 traffic_class = 0;
1024 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1026 if (unformat (input, "version"))
1028 else if (unformat (input, "traffic-class"))
1030 else if (unformat (input, "flow-label"))
1032 else if (unformat (input, "src"))
1034 else if (unformat (input, "dst"))
1036 else if (unformat (input, "proto"))
1039 #define _(a) else if (unformat (input, #a)) a=1;
1040 foreach_ip6_proto_field
1046 #define _(a) found_something += a;
1047 foreach_ip6_proto_field;
1050 if (found_something == 0)
1053 vec_validate (mask, sizeof (*ip) - 1);
1055 ip = (ip6_header_t *) mask;
1057 #define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
1058 foreach_ip6_proto_field;
1061 ip_version_traffic_class_and_flow_label = 0;
1064 ip_version_traffic_class_and_flow_label |= 0xF0000000;
1067 ip_version_traffic_class_and_flow_label |= 0x0FF00000;
1070 ip_version_traffic_class_and_flow_label |= 0x000FFFFF;
1072 ip->ip_version_traffic_class_and_flow_label =
1073 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1080 unformat_l3_mask (unformat_input_t * input, va_list * args)
1082 u8 **maskp = va_arg (*args, u8 **);
1084 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1086 if (unformat (input, "ip4 %U", unformat_ip4_mask, maskp))
1088 else if (unformat (input, "ip6 %U", unformat_ip6_mask, maskp))
1097 unformat_l2_mask (unformat_input_t * input, va_list * args)
1099 u8 **maskp = va_arg (*args, u8 **);
1114 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1116 if (unformat (input, "src"))
1118 else if (unformat (input, "dst"))
1120 else if (unformat (input, "proto"))
1122 else if (unformat (input, "tag1"))
1124 else if (unformat (input, "tag2"))
1126 else if (unformat (input, "ignore-tag1"))
1128 else if (unformat (input, "ignore-tag2"))
1130 else if (unformat (input, "cos1"))
1132 else if (unformat (input, "cos2"))
1134 else if (unformat (input, "dot1q"))
1136 else if (unformat (input, "dot1ad"))
1141 if ((src + dst + proto + tag1 + tag2 + dot1q + dot1ad +
1142 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
1145 if (tag1 || ignore_tag1 || cos1 || dot1q)
1147 if (tag2 || ignore_tag2 || cos2 || dot1ad)
1150 vec_validate (mask, len - 1);
1153 clib_memset (mask, 0xff, 6);
1156 clib_memset (mask + 6, 0xff, 6);
1160 /* inner vlan tag */
1169 mask[21] = mask[20] = 0xff;
1190 mask[16] = mask[17] = 0xff;
1199 mask[12] = mask[13] = 0xff;
1206 unformat_classify_mask (unformat_input_t * input, va_list * args)
1208 u8 **maskp = va_arg (*args, u8 **);
1209 u32 *skipp = va_arg (*args, u32 *);
1210 u32 *matchp = va_arg (*args, u32 *);
1218 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1220 if (unformat (input, "hex %U", unformat_hex_string, &mask))
1222 else if (unformat (input, "l2 %U", unformat_l2_mask, &l2))
1224 else if (unformat (input, "l3 %U", unformat_l3_mask, &l3))
1226 else if (unformat (input, "l4 %U", unformat_l4_mask, &l4))
1240 if (mask || l2 || l3 || l4)
1244 /* "With a free Ethernet header in every package" */
1246 vec_validate (l2, 13);
1250 vec_append (mask, l3);
1255 vec_append (mask, l4);
1260 /* Scan forward looking for the first significant mask octet */
1261 for (i = 0; i < vec_len (mask); i++)
1265 /* compute (skip, match) params */
1266 *skipp = i / sizeof (u32x4);
1267 vec_delete (mask, *skipp * sizeof (u32x4), 0);
1269 /* Pad mask to an even multiple of the vector size */
1270 while (vec_len (mask) % sizeof (u32x4))
1273 match = vec_len (mask) / sizeof (u32x4);
1275 for (i = match * sizeof (u32x4); i > 0; i -= sizeof (u32x4))
1277 u64 *tmp = (u64 *) (mask + (i - sizeof (u32x4)));
1278 if (*tmp || *(tmp + 1))
1283 clib_warning ("BUG: match 0");
1285 _vec_len (mask) = match * sizeof (u32x4);
1296 #define foreach_l2_input_next \
1298 _(ethernet, ETHERNET_INPUT) \
1304 unformat_l2_input_next_index (unformat_input_t * input, va_list * args)
1306 vnet_classify_main_t *cm = &vnet_classify_main;
1307 u32 *miss_next_indexp = va_arg (*args, u32 *);
1312 /* First try registered unformat fns, allowing override... */
1313 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1315 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1323 if (unformat (input, #n)) { next_index = L2_INPUT_CLASSIFY_NEXT_##N; goto out;}
1324 foreach_l2_input_next;
1327 if (unformat (input, "%d", &tmp))
1336 *miss_next_indexp = next_index;
1340 #define foreach_l2_output_next \
1344 unformat_l2_output_next_index (unformat_input_t * input, va_list * args)
1346 vnet_classify_main_t *cm = &vnet_classify_main;
1347 u32 *miss_next_indexp = va_arg (*args, u32 *);
1352 /* First try registered unformat fns, allowing override... */
1353 for (i = 0; i < vec_len (cm->unformat_l2_next_index_fns); i++)
1355 if (unformat (input, "%U", cm->unformat_l2_next_index_fns[i], &tmp))
1363 if (unformat (input, #n)) { next_index = L2_OUTPUT_CLASSIFY_NEXT_##N; goto out;}
1364 foreach_l2_output_next;
1367 if (unformat (input, "%d", &tmp))
1376 *miss_next_indexp = next_index;
1380 #define foreach_ip_next \
1385 unformat_ip_next_index (unformat_input_t * input, va_list * args)
1387 u32 *miss_next_indexp = va_arg (*args, u32 *);
1388 vnet_classify_main_t *cm = &vnet_classify_main;
1393 /* First try registered unformat fns, allowing override... */
1394 for (i = 0; i < vec_len (cm->unformat_ip_next_index_fns); i++)
1396 if (unformat (input, "%U", cm->unformat_ip_next_index_fns[i], &tmp))
1404 if (unformat (input, #n)) { next_index = IP_LOOKUP_NEXT_##N; goto out;}
1408 if (unformat (input, "%d", &tmp))
1417 *miss_next_indexp = next_index;
1421 #define foreach_acl_next \
1425 unformat_acl_next_index (unformat_input_t * input, va_list * args)
1427 u32 *next_indexp = va_arg (*args, u32 *);
1428 vnet_classify_main_t *cm = &vnet_classify_main;
1433 /* First try registered unformat fns, allowing override... */
1434 for (i = 0; i < vec_len (cm->unformat_acl_next_index_fns); i++)
1436 if (unformat (input, "%U", cm->unformat_acl_next_index_fns[i], &tmp))
1444 if (unformat (input, #n)) { next_index = ACL_NEXT_INDEX_##N; goto out;}
1448 if (unformat (input, "permit"))
1453 else if (unformat (input, "%d", &tmp))
1462 *next_indexp = next_index;
1467 unformat_policer_next_index (unformat_input_t * input, va_list * args)
1469 u32 *next_indexp = va_arg (*args, u32 *);
1470 vnet_classify_main_t *cm = &vnet_classify_main;
1475 /* First try registered unformat fns, allowing override... */
1476 for (i = 0; i < vec_len (cm->unformat_policer_next_index_fns); i++)
1479 (input, "%U", cm->unformat_policer_next_index_fns[i], &tmp))
1486 if (unformat (input, "%d", &tmp))
1495 *next_indexp = next_index;
1499 static clib_error_t *
1500 classify_table_command_fn (vlib_main_t * vm,
1501 unformat_input_t * input, vlib_cli_command_t * cmd)
1508 u32 table_index = ~0;
1509 u32 next_table_index = ~0;
1510 u32 miss_next_index = ~0;
1511 u32 memory_size = 2 << 20;
1513 u32 current_data_flag = 0;
1514 int current_data_offset = 0;
1517 vnet_classify_main_t *cm = &vnet_classify_main;
1520 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1522 if (unformat (input, "del"))
1524 else if (unformat (input, "del-chain"))
1529 else if (unformat (input, "buckets %d", &nbuckets))
1531 else if (unformat (input, "skip %d", &skip))
1533 else if (unformat (input, "match %d", &match))
1535 else if (unformat (input, "table %d", &table_index))
1537 else if (unformat (input, "mask %U", unformat_classify_mask,
1538 &mask, &skip, &match))
1540 else if (unformat (input, "memory-size %uM", &tmp))
1541 memory_size = tmp << 20;
1542 else if (unformat (input, "memory-size %uG", &tmp))
1543 memory_size = tmp << 30;
1544 else if (unformat (input, "next-table %d", &next_table_index))
1546 else if (unformat (input, "miss-next %U", unformat_ip_next_index,
1551 (input, "l2-input-miss-next %U", unformat_l2_input_next_index,
1556 (input, "l2-output-miss-next %U", unformat_l2_output_next_index,
1559 else if (unformat (input, "acl-miss-next %U", unformat_acl_next_index,
1562 else if (unformat (input, "current-data-flag %d", ¤t_data_flag))
1565 if (unformat (input, "current-data-offset %d", ¤t_data_offset))
1572 if (is_add && mask == 0 && table_index == ~0)
1573 return clib_error_return (0, "Mask required");
1575 if (is_add && skip == ~0 && table_index == ~0)
1576 return clib_error_return (0, "skip count required");
1578 if (is_add && match == ~0 && table_index == ~0)
1579 return clib_error_return (0, "match count required");
1581 if (!is_add && table_index == ~0)
1582 return clib_error_return (0, "table index required for delete");
1584 rv = vnet_classify_add_del_table (cm, mask, nbuckets, memory_size,
1585 skip, match, next_table_index,
1586 miss_next_index, &table_index,
1587 current_data_flag, current_data_offset,
1595 return clib_error_return (0, "vnet_classify_add_del_table returned %d",
1602 VLIB_CLI_COMMAND (classify_table, static) = {
1603 .path = "classify table",
1605 "classify table [miss-next|l2-miss_next|acl-miss-next <next_index>]"
1606 "\n mask <mask-value> buckets <nn> [skip <n>] [match <n>]"
1607 "\n [current-data-flag <n>] [current-data-offset <n>] [table <n>]"
1608 "\n [memory-size <nn>[M][G]] [next-table <n>]"
1609 "\n [del] [del-chain]",
1610 .function = classify_table_command_fn,
1615 format_vnet_classify_table (u8 * s, va_list * args)
1617 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
1618 int verbose = va_arg (*args, int);
1619 u32 index = va_arg (*args, u32);
1620 vnet_classify_table_t *t;
1624 s = format (s, "%10s%10s%10s%10s", "TableIdx", "Sessions", "NextTbl",
1625 "NextNode", verbose ? "Details" : "");
1629 t = pool_elt_at_index (cm->tables, index);
1630 s = format (s, "%10u%10d%10d%10d", index, t->active_elements,
1631 t->next_table_index, t->miss_next_index);
1633 s = format (s, "\n Heap: %U", format_mheap, t->mheap, 0 /*verbose */ );
1635 s = format (s, "\n nbuckets %d, skip %d match %d flag %d offset %d",
1636 t->nbuckets, t->skip_n_vectors, t->match_n_vectors,
1637 t->current_data_flag, t->current_data_offset);
1638 s = format (s, "\n mask %U", format_hex_bytes, t->mask,
1639 t->match_n_vectors * sizeof (u32x4));
1640 s = format (s, "\n linear-search buckets %d\n", t->linear_buckets);
1645 s = format (s, "\n%U", format_classify_table, t, verbose);
1650 static clib_error_t *
1651 show_classify_tables_command_fn (vlib_main_t * vm,
1652 unformat_input_t * input,
1653 vlib_cli_command_t * cmd)
1655 vnet_classify_main_t *cm = &vnet_classify_main;
1656 vnet_classify_table_t *t;
1657 u32 match_index = ~0;
1662 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1664 if (unformat (input, "index %d", &match_index))
1666 else if (unformat (input, "verbose %d", &verbose))
1668 else if (unformat (input, "verbose"))
1675 pool_foreach (t, cm->tables,
1677 if (match_index == ~0 || (match_index == t - cm->tables))
1678 vec_add1 (indices, t - cm->tables);
1682 if (vec_len (indices))
1684 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm, verbose,
1686 for (i = 0; i < vec_len (indices); i++)
1687 vlib_cli_output (vm, "%U", format_vnet_classify_table, cm,
1688 verbose, indices[i]);
1691 vlib_cli_output (vm, "No classifier tables configured");
1699 VLIB_CLI_COMMAND (show_classify_table_command, static) = {
1700 .path = "show classify tables",
1701 .short_help = "show classify tables [index <nn>]",
1702 .function = show_classify_tables_command_fn,
1707 unformat_l4_match (unformat_input_t * input, va_list * args)
1709 u8 **matchp = va_arg (*args, u8 **);
1711 u8 *proto_header = 0;
1717 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1719 if (unformat (input, "src_port %d", &src_port))
1721 else if (unformat (input, "dst_port %d", &dst_port))
1727 h.src_port = clib_host_to_net_u16 (src_port);
1728 h.dst_port = clib_host_to_net_u16 (dst_port);
1729 vec_validate (proto_header, sizeof (h) - 1);
1730 memcpy (proto_header, &h, sizeof (h));
1732 *matchp = proto_header;
1738 unformat_ip4_match (unformat_input_t * input, va_list * args)
1740 u8 **matchp = va_arg (*args, u8 **);
1747 int src = 0, dst = 0;
1748 ip4_address_t src_val, dst_val;
1755 int fragment_id = 0;
1756 u32 fragment_id_val;
1762 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1764 if (unformat (input, "version %d", &version_val))
1766 else if (unformat (input, "hdr_length %d", &hdr_length_val))
1768 else if (unformat (input, "src %U", unformat_ip4_address, &src_val))
1770 else if (unformat (input, "dst %U", unformat_ip4_address, &dst_val))
1772 else if (unformat (input, "proto %d", &proto_val))
1774 else if (unformat (input, "tos %d", &tos_val))
1776 else if (unformat (input, "length %d", &length_val))
1778 else if (unformat (input, "fragment_id %d", &fragment_id_val))
1780 else if (unformat (input, "ttl %d", &ttl_val))
1782 else if (unformat (input, "checksum %d", &checksum_val))
1788 if (version + hdr_length + src + dst + proto + tos + length + fragment_id
1789 + ttl + checksum == 0)
1793 * Aligned because we use the real comparison functions
1795 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
1797 ip = (ip4_header_t *) match;
1799 /* These are realistically matched in practice */
1801 ip->src_address.as_u32 = src_val.as_u32;
1804 ip->dst_address.as_u32 = dst_val.as_u32;
1807 ip->protocol = proto_val;
1810 /* These are not, but they're included for completeness */
1812 ip->ip_version_and_header_length |= (version_val & 0xF) << 4;
1815 ip->ip_version_and_header_length |= (hdr_length_val & 0xF);
1821 ip->length = clib_host_to_net_u16 (length_val);
1827 ip->checksum = clib_host_to_net_u16 (checksum_val);
1834 unformat_ip6_match (unformat_input_t * input, va_list * args)
1836 u8 **matchp = va_arg (*args, u8 **);
1841 u8 traffic_class = 0;
1842 u32 traffic_class_val;
1845 int src = 0, dst = 0;
1846 ip6_address_t src_val, dst_val;
1849 int payload_length = 0;
1850 u32 payload_length_val;
1853 u32 ip_version_traffic_class_and_flow_label;
1855 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1857 if (unformat (input, "version %d", &version_val))
1859 else if (unformat (input, "traffic_class %d", &traffic_class_val))
1861 else if (unformat (input, "flow_label %d", &flow_label_val))
1863 else if (unformat (input, "src %U", unformat_ip6_address, &src_val))
1865 else if (unformat (input, "dst %U", unformat_ip6_address, &dst_val))
1867 else if (unformat (input, "proto %d", &proto_val))
1869 else if (unformat (input, "payload_length %d", &payload_length_val))
1871 else if (unformat (input, "hop_limit %d", &hop_limit_val))
1877 if (version + traffic_class + flow_label + src + dst + proto +
1878 payload_length + hop_limit == 0)
1882 * Aligned because we use the real comparison functions
1884 vec_validate_aligned (match, sizeof (*ip) - 1, sizeof (u32x4));
1886 ip = (ip6_header_t *) match;
1889 clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address));
1892 clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
1895 ip->protocol = proto_val;
1897 ip_version_traffic_class_and_flow_label = 0;
1900 ip_version_traffic_class_and_flow_label |= (version_val & 0xF) << 28;
1903 ip_version_traffic_class_and_flow_label |=
1904 (traffic_class_val & 0xFF) << 20;
1907 ip_version_traffic_class_and_flow_label |= (flow_label_val & 0xFFFFF);
1909 ip->ip_version_traffic_class_and_flow_label =
1910 clib_host_to_net_u32 (ip_version_traffic_class_and_flow_label);
1913 ip->payload_length = clib_host_to_net_u16 (payload_length_val);
1916 ip->hop_limit = hop_limit_val;
1923 unformat_l3_match (unformat_input_t * input, va_list * args)
1925 u8 **matchp = va_arg (*args, u8 **);
1927 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1929 if (unformat (input, "ip4 %U", unformat_ip4_match, matchp))
1931 else if (unformat (input, "ip6 %U", unformat_ip6_match, matchp))
1941 unformat_vlan_tag (unformat_input_t * input, va_list * args)
1943 u8 *tagp = va_arg (*args, u8 *);
1946 if (unformat (input, "%d", &tag))
1948 tagp[0] = (tag >> 8) & 0x0F;
1949 tagp[1] = tag & 0xFF;
1957 unformat_l2_match (unformat_input_t * input, va_list * args)
1959 u8 **matchp = va_arg (*args, u8 **);
1979 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1981 if (unformat (input, "src %U", unformat_ethernet_address, &src_val))
1984 if (unformat (input, "dst %U", unformat_ethernet_address, &dst_val))
1986 else if (unformat (input, "proto %U",
1987 unformat_ethernet_type_host_byte_order, &proto_val))
1989 else if (unformat (input, "tag1 %U", unformat_vlan_tag, tag1_val))
1991 else if (unformat (input, "tag2 %U", unformat_vlan_tag, tag2_val))
1993 else if (unformat (input, "ignore-tag1"))
1995 else if (unformat (input, "ignore-tag2"))
1997 else if (unformat (input, "cos1 %d", &cos1_val))
1999 else if (unformat (input, "cos2 %d", &cos2_val))
2004 if ((src + dst + proto + tag1 + tag2 +
2005 ignore_tag1 + ignore_tag2 + cos1 + cos2) == 0)
2008 if (tag1 || ignore_tag1 || cos1)
2010 if (tag2 || ignore_tag2 || cos2)
2013 vec_validate_aligned (match, len - 1, sizeof (u32x4));
2016 clib_memcpy_fast (match, dst_val, 6);
2019 clib_memcpy_fast (match + 6, src_val, 6);
2023 /* inner vlan tag */
2024 match[19] = tag2_val[1];
2025 match[18] = tag2_val[0];
2027 match[18] |= (cos2_val & 0x7) << 5;
2030 match[21] = proto_val & 0xff;
2031 match[20] = proto_val >> 8;
2035 match[15] = tag1_val[1];
2036 match[14] = tag1_val[0];
2039 match[14] |= (cos1_val & 0x7) << 5;
2045 match[15] = tag1_val[1];
2046 match[14] = tag1_val[0];
2049 match[17] = proto_val & 0xff;
2050 match[16] = proto_val >> 8;
2053 match[14] |= (cos1_val & 0x7) << 5;
2059 match[18] |= (cos2_val & 0x7) << 5;
2061 match[14] |= (cos1_val & 0x7) << 5;
2064 match[13] = proto_val & 0xff;
2065 match[12] = proto_val >> 8;
2074 unformat_classify_match (unformat_input_t * input, va_list * args)
2076 vnet_classify_main_t *cm = va_arg (*args, vnet_classify_main_t *);
2077 u8 **matchp = va_arg (*args, u8 **);
2078 u32 table_index = va_arg (*args, u32);
2079 vnet_classify_table_t *t;
2086 if (pool_is_free_index (cm->tables, table_index))
2089 t = pool_elt_at_index (cm->tables, table_index);
2091 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2093 if (unformat (input, "hex %U", unformat_hex_string, &match))
2095 else if (unformat (input, "l2 %U", unformat_l2_match, &l2))
2097 else if (unformat (input, "l3 %U", unformat_l3_match, &l3))
2099 else if (unformat (input, "l4 %U", unformat_l4_match, &l4))
2113 if (match || l2 || l3 || l4)
2117 /* "Win a free Ethernet header in every packet" */
2119 vec_validate_aligned (l2, 13, sizeof (u32x4));
2123 vec_append_aligned (match, l3, sizeof (u32x4));
2128 vec_append_aligned (match, l4, sizeof (u32x4));
2133 /* Make sure the vector is big enough even if key is all 0's */
2134 vec_validate_aligned
2136 ((t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4)) - 1,
2139 /* Set size, include skipped vectors */
2141 (t->match_n_vectors + t->skip_n_vectors) * sizeof (u32x4);
2152 vnet_classify_add_del_session (vnet_classify_main_t * cm,
2158 u8 action, u32 metadata, int is_add)
2160 vnet_classify_table_t *t;
2161 vnet_classify_entry_5_t _max_e __attribute__ ((aligned (16)));
2162 vnet_classify_entry_t *e;
2165 if (pool_is_free_index (cm->tables, table_index))
2166 return VNET_API_ERROR_NO_SUCH_TABLE;
2168 t = pool_elt_at_index (cm->tables, table_index);
2170 e = (vnet_classify_entry_t *) & _max_e;
2171 e->next_index = hit_next_index;
2172 e->opaque_index = opaque_index;
2173 e->advance = advance;
2178 if (e->action == CLASSIFY_ACTION_SET_IP4_FIB_INDEX)
2179 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
2181 FIB_SOURCE_CLASSIFY);
2182 else if (e->action == CLASSIFY_ACTION_SET_IP6_FIB_INDEX)
2183 e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
2185 FIB_SOURCE_CLASSIFY);
2186 else if (e->action == CLASSIFY_ACTION_SET_METADATA)
2187 e->metadata = metadata;
2191 /* Copy key data, honoring skip_n_vectors */
2192 clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
2193 t->match_n_vectors * sizeof (u32x4));
2195 /* Clear don't-care bits; likely when dynamically creating sessions */
2196 for (i = 0; i < t->match_n_vectors; i++)
2197 e->key[i] &= t->mask[i];
2199 rv = vnet_classify_add_del (t, e, is_add);
2201 vnet_classify_entry_release_resource (e);
2204 return VNET_API_ERROR_NO_SUCH_ENTRY;
2208 static clib_error_t *
2209 classify_session_command_fn (vlib_main_t * vm,
2210 unformat_input_t * input,
2211 vlib_cli_command_t * cmd)
2213 vnet_classify_main_t *cm = &vnet_classify_main;
2215 u32 table_index = ~0;
2216 u32 hit_next_index = ~0;
2217 u64 opaque_index = ~0;
2224 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2226 if (unformat (input, "del"))
2228 else if (unformat (input, "hit-next %U", unformat_ip_next_index,
2233 (input, "l2-input-hit-next %U", unformat_l2_input_next_index,
2238 (input, "l2-output-hit-next %U", unformat_l2_output_next_index,
2241 else if (unformat (input, "acl-hit-next %U", unformat_acl_next_index,
2244 else if (unformat (input, "policer-hit-next %U",
2245 unformat_policer_next_index, &hit_next_index))
2247 else if (unformat (input, "opaque-index %lld", &opaque_index))
2249 else if (unformat (input, "match %U", unformat_classify_match,
2250 cm, &match, table_index))
2252 else if (unformat (input, "advance %d", &advance))
2254 else if (unformat (input, "table-index %d", &table_index))
2256 else if (unformat (input, "action set-ip4-fib-id %d", &metadata))
2258 else if (unformat (input, "action set-ip6-fib-id %d", &metadata))
2260 else if (unformat (input, "action set-sr-policy-index %d", &metadata))
2264 /* Try registered opaque-index unformat fns */
2265 for (i = 0; i < vec_len (cm->unformat_opaque_index_fns); i++)
2267 if (unformat (input, "%U", cm->unformat_opaque_index_fns[i],
2277 if (table_index == ~0)
2278 return clib_error_return (0, "Table index required");
2280 if (is_add && match == 0)
2281 return clib_error_return (0, "Match value required");
2283 rv = vnet_classify_add_del_session (cm, table_index, match,
2285 opaque_index, advance,
2286 action, metadata, is_add);
2294 return clib_error_return (0,
2295 "vnet_classify_add_del_session returned %d",
2303 VLIB_CLI_COMMAND (classify_session_command, static) = {
2304 .path = "classify session",
2306 "classify session [hit-next|l2-input-hit-next|l2-output-hit-next|"
2307 "acl-hit-next <next_index>|policer-hit-next <policer_name>]"
2308 "\n table-index <nn> match [hex] [l2] [l3 ip4] [opaque-index <index>]"
2309 "\n [action set-ip4-fib-id|set-ip6-fib-id|set-sr-policy-index <n>] [del]",
2310 .function = classify_session_command_fn,
2315 unformat_opaque_sw_if_index (unformat_input_t * input, va_list * args)
2317 u64 *opaquep = va_arg (*args, u64 *);
2320 if (unformat (input, "opaque-sw_if_index %U", unformat_vnet_sw_interface,
2321 vnet_get_main (), &sw_if_index))
2323 *opaquep = sw_if_index;
2330 unformat_ip_next_node (unformat_input_t * input, va_list * args)
2332 vnet_classify_main_t *cm = &vnet_classify_main;
2333 u32 *next_indexp = va_arg (*args, u32 *);
2335 u32 next_index = ~0;
2337 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2338 cm->vlib_main, &node_index))
2340 next_index = vlib_node_add_next (cm->vlib_main,
2341 ip6_classify_node.index, node_index);
2343 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2344 cm->vlib_main, &node_index))
2346 next_index = vlib_node_add_next (cm->vlib_main,
2347 ip4_classify_node.index, node_index);
2352 *next_indexp = next_index;
2357 unformat_acl_next_node (unformat_input_t * input, va_list * args)
2359 vnet_classify_main_t *cm = &vnet_classify_main;
2360 u32 *next_indexp = va_arg (*args, u32 *);
2364 if (unformat (input, "ip6-node %U", unformat_vlib_node,
2365 cm->vlib_main, &node_index))
2367 next_index = vlib_node_add_next (cm->vlib_main,
2368 ip6_inacl_node.index, node_index);
2370 else if (unformat (input, "ip4-node %U", unformat_vlib_node,
2371 cm->vlib_main, &node_index))
2373 next_index = vlib_node_add_next (cm->vlib_main,
2374 ip4_inacl_node.index, node_index);
2379 *next_indexp = next_index;
2384 unformat_l2_input_next_node (unformat_input_t * input, va_list * args)
2386 vnet_classify_main_t *cm = &vnet_classify_main;
2387 u32 *next_indexp = va_arg (*args, u32 *);
2391 if (unformat (input, "input-node %U", unformat_vlib_node,
2392 cm->vlib_main, &node_index))
2394 next_index = vlib_node_add_next
2395 (cm->vlib_main, l2_input_classify_node.index, node_index);
2397 *next_indexp = next_index;
2404 unformat_l2_output_next_node (unformat_input_t * input, va_list * args)
2406 vnet_classify_main_t *cm = &vnet_classify_main;
2407 u32 *next_indexp = va_arg (*args, u32 *);
2411 if (unformat (input, "output-node %U", unformat_vlib_node,
2412 cm->vlib_main, &node_index))
2414 next_index = vlib_node_add_next
2415 (cm->vlib_main, l2_output_classify_node.index, node_index);
2417 *next_indexp = next_index;
2423 static clib_error_t *
2424 vnet_classify_init (vlib_main_t * vm)
2426 vnet_classify_main_t *cm = &vnet_classify_main;
2429 cm->vnet_main = vnet_get_main ();
2431 vnet_classify_register_unformat_opaque_index_fn
2432 (unformat_opaque_sw_if_index);
2434 vnet_classify_register_unformat_ip_next_index_fn (unformat_ip_next_node);
2436 vnet_classify_register_unformat_l2_next_index_fn
2437 (unformat_l2_input_next_node);
2439 vnet_classify_register_unformat_l2_next_index_fn
2440 (unformat_l2_output_next_node);
2442 vnet_classify_register_unformat_acl_next_index_fn (unformat_acl_next_node);
2447 VLIB_INIT_FUNCTION (vnet_classify_init);
2461 test_entry_t *entries;
2463 /* test parameters */
2469 vnet_classify_table_t *table;
2477 classify_data_or_mask_t *mask;
2478 classify_data_or_mask_t *data;
2481 vnet_classify_main_t *classify_main;
2482 vlib_main_t *vlib_main;
2484 } test_classify_main_t;
2486 static test_classify_main_t test_classify_main;
2488 static clib_error_t *
2489 test_classify_churn (test_classify_main_t * tm)
2491 classify_data_or_mask_t *mask, *data;
2492 vlib_main_t *vm = tm->vlib_main;
2494 u8 *mp = 0, *dp = 0;
2498 vec_validate_aligned (mp, 3 * sizeof (u32x4), sizeof (u32x4));
2499 vec_validate_aligned (dp, 3 * sizeof (u32x4), sizeof (u32x4));
2501 mask = (classify_data_or_mask_t *) mp;
2502 data = (classify_data_or_mask_t *) dp;
2504 /* Mask on src address */
2505 clib_memset (&mask->ip.src_address, 0xff, 4);
2507 tmp = clib_host_to_net_u32 (tm->src.as_u32);
2509 for (i = 0; i < tm->sessions; i++)
2511 vec_add2 (tm->entries, ep, 1);
2512 ep->addr.as_u32 = clib_host_to_net_u32 (tmp);
2517 tm->table = vnet_classify_new_table (tm->classify_main,
2520 tm->memory_size, 0 /* skip */ ,
2521 3 /* vectors to match */ );
2522 tm->table->miss_next_index = IP_LOOKUP_NEXT_DROP;
2523 tm->table_index = tm->table - tm->classify_main->tables;
2524 vlib_cli_output (vm, "Created table %d, buckets %d",
2525 tm->table_index, tm->buckets);
2527 vlib_cli_output (vm, "Initialize: add %d (approx. half of %d sessions)...",
2528 tm->sessions / 2, tm->sessions);
2530 for (i = 0; i < tm->sessions / 2; i++)
2532 ep = vec_elt_at_index (tm->entries, i);
2534 data->ip.src_address.as_u32 = ep->addr.as_u32;
2537 rv = vnet_classify_add_del_session (tm->classify_main,
2540 IP_LOOKUP_NEXT_DROP,
2541 i /* opaque_index */ ,
2548 clib_warning ("add: returned %d", rv);
2551 vlib_cli_output (vm, "add: %U", format_ip4_address, &ep->addr.as_u32);
2554 vlib_cli_output (vm, "Execute %d random add/delete operations",
2557 for (i = 0; i < tm->iterations; i++)
2561 /* Pick a random entry */
2562 index = random_u32 (&tm->seed) % tm->sessions;
2564 ep = vec_elt_at_index (tm->entries, index);
2566 data->ip.src_address.as_u32 = ep->addr.as_u32;
2568 /* If it's in the table, remove it. Else, add it */
2569 is_add = !ep->in_table;
2572 vlib_cli_output (vm, "%s: %U",
2573 is_add ? "add" : "del",
2574 format_ip4_address, &ep->addr.as_u32);
2576 rv = vnet_classify_add_del_session (tm->classify_main,
2579 IP_LOOKUP_NEXT_DROP,
2580 i /* opaque_index */ ,
2586 vlib_cli_output (vm,
2587 "%s[%d]: %U returned %d", is_add ? "add" : "del",
2588 index, format_ip4_address, &ep->addr.as_u32, rv);
2590 ep->in_table = is_add;
2593 vlib_cli_output (vm, "Remove remaining %d entries from the table",
2594 tm->table->active_elements);
2596 for (i = 0; i < tm->sessions; i++)
2600 vnet_classify_entry_t *e;
2602 ep = tm->entries + i;
2603 if (ep->in_table == 0)
2606 data->ip.src_address.as_u32 = ep->addr.as_u32;
2608 hash = vnet_classify_hash_packet (tm->table, (u8 *) data);
2610 e = vnet_classify_find_entry (tm->table,
2611 (u8 *) data, hash, 0 /* time_now */ );
2614 clib_warning ("Couldn't find %U index %d which should be present",
2615 format_ip4_address, ep->addr, i);
2619 key_minus_skip = (u8 *) e->key;
2620 key_minus_skip -= tm->table->skip_n_vectors * sizeof (u32x4);
2622 rv = vnet_classify_add_del_session
2625 key_minus_skip, IP_LOOKUP_NEXT_DROP, i /* opaque_index */ ,
2626 0 /* advance */ , 0, 0,
2630 clib_warning ("del: returned %d", rv);
2633 vlib_cli_output (vm, "del: %U", format_ip4_address, &ep->addr.as_u32);
2636 vlib_cli_output (vm, "%d entries remain, MUST be zero",
2637 tm->table->active_elements);
2639 vlib_cli_output (vm, "Table after cleanup: \n%U\n",
2640 format_classify_table, tm->table, 0 /* verbose */ );
2645 vnet_classify_delete_table_index (tm->classify_main,
2646 tm->table_index, 1 /* del_chain */ );
2648 tm->table_index = ~0;
2649 vec_free (tm->entries);
2654 static clib_error_t *
2655 test_classify_command_fn (vlib_main_t * vm,
2656 unformat_input_t * input, vlib_cli_command_t * cmd)
2658 test_classify_main_t *tm = &test_classify_main;
2659 vnet_classify_main_t *cm = &vnet_classify_main;
2662 clib_error_t *error = 0;
2665 tm->sessions = 8192;
2666 tm->iterations = 8192;
2667 tm->memory_size = 64 << 20;
2668 tm->src.as_u32 = clib_net_to_host_u32 (0x0100000A);
2670 tm->seed = 0xDEADDABE;
2671 tm->classify_main = cm;
2675 /* Default starting address 1.0.0.10 */
2677 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2679 if (unformat (input, "sessions %d", &tmp))
2682 if (unformat (input, "src %U", unformat_ip4_address, &tm->src.as_u32))
2684 else if (unformat (input, "buckets %d", &tm->buckets))
2686 else if (unformat (input, "memory-size %uM", &tmp))
2687 tm->memory_size = tmp << 20;
2688 else if (unformat (input, "memory-size %uG", &tmp))
2689 tm->memory_size = tmp << 30;
2690 else if (unformat (input, "seed %d", &tm->seed))
2692 else if (unformat (input, "verbose"))
2695 else if (unformat (input, "iterations %d", &tm->iterations))
2697 else if (unformat (input, "churn-test"))
2706 error = test_classify_churn (tm);
2709 error = clib_error_return (0, "No such test");
2717 VLIB_CLI_COMMAND (test_classify_command, static) = {
2718 .path = "test classify",
2720 "test classify [src <ip>] [sessions <nn>] [buckets <nn>] [seed <nnn>]\n"
2721 " [memory-size <nn>[M|G]]\n"
2723 .function = test_classify_command_fn,
2726 #endif /* TEST_CODE */
2729 * fd.io coding-style-patch-verification: ON
2732 * eval: (c-set-style "gnu")