2 * Copyright (c) 2016 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <vnet/lisp-gpe/lisp_gpe.h>
18 /* avoids calling route callbacks for src fib */
20 ip4_sd_fib_set_adj_index (lisp_gpe_main_t * lgm, ip4_fib_t * fib, u32 flags,
21 u32 dst_address_u32, u32 dst_address_length,
24 ip_lookup_main_t *lm = lgm->lm4;
27 if (vec_bytes (fib->old_hash_values))
28 memset (fib->old_hash_values, ~0, vec_bytes (fib->old_hash_values));
29 if (vec_bytes (fib->new_hash_values))
30 memset (fib->new_hash_values, ~0, vec_bytes (fib->new_hash_values));
31 fib->new_hash_values[0] = adj_index;
33 /* Make sure adj index is valid. */
35 (void) ip_get_adjacency (lm, adj_index);
37 hash = fib->adj_index_by_dst_address[dst_address_length];
39 hash = _hash_set3 (hash, dst_address_u32,
40 fib->new_hash_values, fib->old_hash_values);
42 fib->adj_index_by_dst_address[dst_address_length] = hash;
45 /* copied from ip4_forward since it's static */
47 ip4_fib_init_adj_index_by_dst_address (ip_lookup_main_t * lm,
48 ip4_fib_t * fib, u32 address_length)
53 ASSERT (lm->fib_result_n_bytes >= sizeof (uword));
54 lm->fib_result_n_words = round_pow2 (lm->fib_result_n_bytes, sizeof (uword))
57 fib->adj_index_by_dst_address[address_length] =
58 hash_create (32 /* elts */ , lm->fib_result_n_words * sizeof (uword));
60 hash_set_flags (fib->adj_index_by_dst_address[address_length],
61 HASH_FLAG_NO_AUTO_SHRINK);
63 h = hash_header (fib->adj_index_by_dst_address[address_length]);
64 max_index = (hash_value_bytes (h) / sizeof (fib->new_hash_values[0])) - 1;
66 /* Initialize new/old hash value vectors. */
67 vec_validate_init_empty (fib->new_hash_values, max_index, ~0);
68 vec_validate_init_empty (fib->old_hash_values, max_index, ~0);
72 ip4_sd_fib_add_del_src_route (lisp_gpe_main_t * lgm,
73 ip4_add_del_route_args_t * a)
75 ip_lookup_main_t *lm = lgm->lm4;
77 u32 dst_address, dst_address_length, adj_index, old_adj_index;
80 /* Either create new adjacency or use given one depending on arguments. */
82 ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
84 adj_index = a->adj_index;
86 dst_address = a->dst_address.data_u32;
87 dst_address_length = a->dst_address_length;
89 fib = pool_elt_at_index (lgm->ip4_src_fibs, a->table_index_or_table_id);
91 if (!fib->adj_index_by_dst_address[dst_address_length])
92 ip4_fib_init_adj_index_by_dst_address (lm, fib, dst_address_length);
94 hash = fib->adj_index_by_dst_address[dst_address_length];
96 is_del = (a->flags & IP4_ROUTE_FLAG_DEL) != 0;
100 fib->old_hash_values[0] = ~0;
101 hash = _hash_unset (hash, dst_address, fib->old_hash_values);
102 fib->adj_index_by_dst_address[dst_address_length] = hash;
105 ip4_sd_fib_set_adj_index (lgm, fib, a->flags, dst_address,
106 dst_address_length, adj_index);
108 old_adj_index = fib->old_hash_values[0];
110 /* Avoid spurious reference count increments */
111 if (old_adj_index == adj_index
112 && adj_index != ~0 && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
114 ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
115 if (adj->share_count > 0)
119 ip4_fib_mtrie_add_del_route (fib, a->dst_address, dst_address_length,
120 is_del ? old_adj_index : adj_index, is_del);
122 /* Delete old adjacency index if present and changed. */
123 if (!(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
124 && old_adj_index != ~0 && old_adj_index != adj_index)
125 ip_del_adjacency (lm, old_adj_index);
129 ip4_sd_get_src_route (lisp_gpe_main_t * lgm, u32 src_fib_index,
130 ip4_address_t * src, u32 address_length)
132 ip4_fib_t *fib = pool_elt_at_index (lgm->ip4_src_fibs, src_fib_index);
135 hash = fib->adj_index_by_dst_address[address_length];
136 p = hash_get (hash, src->as_u32);
141 typedef CLIB_PACKED (struct ip4_route {
142 ip4_address_t address;
143 u32 address_length : 6;
149 ip4_sd_fib_clear_src_fib (lisp_gpe_main_t * lgm, ip4_fib_t * fib)
151 ip4_route_t *routes = 0, *r;
154 vec_reset_length (routes);
156 for (i = 0; i < ARRAY_LEN (fib->adj_index_by_dst_address); i++)
158 uword *hash = fib->adj_index_by_dst_address[i];
162 x.address_length = i;
163 x.index = 0; /* shut up coverity */
166 hash_foreach_pair (p, hash,
168 x.address.data_u32 = p->key;
169 vec_add1 (routes, x);
174 vec_foreach (r, routes)
176 ip4_add_del_route_args_t a;
178 memset (&a, 0, sizeof (a));
179 a.flags = IP4_ROUTE_FLAG_FIB_INDEX | IP4_ROUTE_FLAG_DEL;
180 a.table_index_or_table_id = fib - lgm->ip4_src_fibs;
181 a.dst_address = r->address;
182 a.dst_address_length = r->address_length;
185 ip4_sd_fib_add_del_src_route (lgm, &a);
190 ip4_fib_is_empty (ip4_fib_t * fib)
196 for (i = ARRAY_LEN (fib->adj_index_by_dst_address) - 1; i >= 0; i--)
198 uword *hash = fib->adj_index_by_dst_address[i];
199 uword n_elts = hash_elts (hash);
210 ip4_sd_fib_add_del_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
211 ip_prefix_t * src_prefix, u32 table_id,
212 ip_adjacency_t * add_adj, u8 is_add)
215 ip4_add_del_route_args_t a;
216 ip_adjacency_t *dst_adjp, dst_adj;
217 ip4_address_t dst = ip_prefix_v4 (dst_prefix), src;
218 u32 dst_address_length = ip_prefix_len (dst_prefix), src_address_length = 0;
223 src = ip_prefix_v4 (src_prefix);
224 src_address_length = ip_prefix_len (src_prefix);
227 memset (&src, 0, sizeof (src));
230 p = ip4_get_route (lgm->im4, table_id, 0, dst.as_u8, dst_address_length);
234 /* insert dst prefix to ip4 fib, if it's not in yet */
237 /* allocate and init src ip4 fib */
238 pool_get (lgm->ip4_src_fibs, src_fib);
239 ip4_mtrie_init (&src_fib->mtrie);
241 /* configure adjacency */
242 memset (&dst_adj, 0, sizeof (dst_adj));
244 /* reuse rewrite header to store pointer to src fib */
245 dst_adj.rewrite_header.sw_if_index = src_fib - lgm->ip4_src_fibs;
247 /* dst adj should point to lisp gpe lookup */
248 dst_adj.lookup_next_index = lgm->ip4_lookup_next_lgpe_ip4_lookup;
250 /* explicit_fib_index is used in IP6 FIB lookup, don't reuse it */
251 dst_adj.explicit_fib_index = ~0;
254 /* make sure we have different signatures for adj in different tables
255 * but with the same lookup_next_index and for adj in the same table
256 * but associated to different destinations */
257 dst_adj.if_address_index = table_id;
258 dst_adj.indirect.next_hop.ip4 = dst;
260 memset (&a, 0, sizeof (a));
261 a.flags = IP4_ROUTE_FLAG_TABLE_ID;
262 a.table_index_or_table_id = table_id; /* vrf */
264 a.dst_address_length = dst_address_length;
266 a.flags |= IP4_ROUTE_FLAG_ADD;
267 a.add_adj = &dst_adj;
270 ip4_add_del_route (lgm->im4, &a);
272 /* lookup dst adj to obtain the adj index */
273 p = ip4_get_route (lgm->im4, table_id, 0, dst.as_u8,
276 /* make sure insertion succeeded */
280 dst_adjp = ip_get_adjacency (lgm->lm4, p[0]);
281 ASSERT (dst_adjp->rewrite_header.sw_if_index
282 == dst_adj.rewrite_header.sw_if_index);
291 ("Trying to delete inexistent dst route for %U. Aborting",
292 format_ip4_address_and_length, dst.as_u8, dst_address_length);
297 dst_adjp = ip_get_adjacency (lgm->lm4, p[0]);
299 /* add/del src prefix to src fib */
300 memset (&a, 0, sizeof (a));
301 a.flags = IP4_ROUTE_FLAG_TABLE_ID;
302 a.table_index_or_table_id = dst_adjp->rewrite_header.sw_if_index;
304 a.flags |= is_add ? IP4_ROUTE_FLAG_ADD : IP4_ROUTE_FLAG_DEL;
306 a.n_add_adj = is_add ? 1 : 0;
307 /* if src prefix is null, add 0/0 */
308 a.dst_address_length = src_address_length;
310 ip4_sd_fib_add_del_src_route (lgm, &a);
312 /* make sure insertion succeeded */
313 if (CLIB_DEBUG && is_add)
316 ip_adjacency_t *src_adjp;
317 sai = ip4_sd_get_src_route (lgm, dst_adjp->rewrite_header.sw_if_index,
318 &src, src_address_length);
320 src_adjp = ip_get_adjacency (lgm->lm4, sai[0]);
321 ASSERT (src_adjp->if_address_index == add_adj->if_address_index);
324 /* if a delete, check if there are elements left in the src fib */
327 src_fib = pool_elt_at_index (lgm->ip4_src_fibs,
328 dst_adjp->rewrite_header.sw_if_index);
332 /* if there's nothing left */
333 if (ip4_fib_is_empty (src_fib))
335 /* remove the src fib .. */
336 pool_put (lgm->ip4_src_fibs, src_fib);
338 /* .. and remove dst route */
339 memset (&a, 0, sizeof (a));
340 a.flags = IP4_ROUTE_FLAG_TABLE_ID;
341 a.table_index_or_table_id = table_id; /* vrf */
343 a.dst_address_length = dst_address_length;
345 a.flags |= IP4_ROUTE_FLAG_DEL;
347 ip4_add_del_route (lgm->im4, &a);
355 ip4_sd_fib_get_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
356 ip_prefix_t * src_prefix, u32 table_id)
359 ip4_address_t dst = ip_prefix_v4 (dst_prefix), src;
360 u32 dst_address_length = ip_prefix_len (dst_prefix), src_address_length = 0;
361 ip_adjacency_t *dst_adj;
365 src = ip_prefix_v4 (src_prefix);
366 src_address_length = ip_prefix_len (src_prefix);
369 memset (&src, 0, sizeof (src));
372 p = ip4_get_route (lgm->im4, table_id, 0, dst.as_u8, dst_address_length);
376 dst_adj = ip_get_adjacency (lgm->lm4, p[0]);
377 return ip4_sd_get_src_route (lgm, dst_adj->rewrite_header.sw_if_index, &src,
382 ip6_sd_get_src_route (lisp_gpe_main_t * lgm, u32 src_fib_index,
383 ip6_address_t * src, u32 address_length)
386 BVT (clib_bihash_kv) kv, value;
387 ip6_src_fib_t *fib = pool_elt_at_index (lgm->ip6_src_fibs, src_fib_index);
391 ASSERT (address_length <= 128);
393 mask = &fib->fib_masks[address_length];
395 kv.key[0] = src->as_u64[0] & mask->as_u64[0];
396 kv.key[1] = src->as_u64[1] & mask->as_u64[1];
397 kv.key[2] = address_length;
399 rv = BV (clib_bihash_search_inline_2) (&fib->ip6_lookup_table, &kv, &value);
407 compute_prefix_lengths_in_search_order (ip6_src_fib_t * fib)
410 vec_reset_length (fib->prefix_lengths_in_search_order);
411 /* Note: bitmap reversed so this is in fact a longest prefix match */
414 clib_bitmap_foreach(i, fib->non_empty_dst_address_length_bitmap, ({
415 int dst_address_length = 128 - i;
416 vec_add1 (fib->prefix_lengths_in_search_order, dst_address_length);
421 /* Rewrite of ip6_add_del_route() because it uses im6 to find the fib */
423 ip6_sd_fib_add_del_src_route (lisp_gpe_main_t * lgm,
424 ip6_add_del_route_args_t * a)
426 ip_lookup_main_t *lm = lgm->lm6;
428 ip6_address_t dst_address;
429 u32 dst_address_length, adj_index;
431 u32 old_adj_index = ~0;
432 BVT (clib_bihash_kv) kv, value;
434 vlib_smp_unsafe_warning ();
436 is_del = (a->flags & IP6_ROUTE_FLAG_DEL) != 0;
438 /* Either create new adjacency or use given one depending on arguments. */
439 if (a->n_add_adj > 0)
441 ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
444 adj_index = a->adj_index;
446 dst_address = a->dst_address;
447 dst_address_length = a->dst_address_length;
448 fib = pool_elt_at_index (lgm->ip6_src_fibs, a->table_index_or_table_id);
450 ASSERT (dst_address_length < ARRAY_LEN (fib->fib_masks));
451 ip6_address_mask (&dst_address, &fib->fib_masks[dst_address_length]);
453 /* refcount accounting */
456 ASSERT (fib->dst_address_length_refcounts[dst_address_length] > 0);
457 if (--fib->dst_address_length_refcounts[dst_address_length] == 0)
459 fib->non_empty_dst_address_length_bitmap =
460 clib_bitmap_set (fib->non_empty_dst_address_length_bitmap,
461 128 - dst_address_length, 0);
462 compute_prefix_lengths_in_search_order (fib);
467 fib->dst_address_length_refcounts[dst_address_length]++;
469 fib->non_empty_dst_address_length_bitmap =
470 clib_bitmap_set (fib->non_empty_dst_address_length_bitmap,
471 128 - dst_address_length, 1);
472 compute_prefix_lengths_in_search_order (fib);
475 kv.key[0] = dst_address.as_u64[0];
476 kv.key[1] = dst_address.as_u64[1];
477 kv.key[2] = dst_address_length;
479 if (BV (clib_bihash_search) (&fib->ip6_lookup_table, &kv, &value) == 0)
480 old_adj_index = value.value;
483 BV (clib_bihash_add_del) (&fib->ip6_lookup_table, &kv, 0 /* is_add */ );
486 /* Make sure adj index is valid. */
488 (void) ip_get_adjacency (lm, adj_index);
490 kv.value = adj_index;
492 BV (clib_bihash_add_del) (&fib->ip6_lookup_table, &kv, 1 /* is_add */ );
495 /* Avoid spurious reference count increments */
496 if (old_adj_index == adj_index
497 && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
499 ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
500 if (adj->share_count > 0)
504 /* Delete old adjacency index if present and changed. */
506 if (!(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
507 && old_adj_index != ~0 && old_adj_index != adj_index)
508 ip_del_adjacency (lm, old_adj_index);
513 ip6_src_fib_init (ip6_src_fib_t * fib)
517 for (i = 0; i < ARRAY_LEN (fib->fib_masks); i++)
524 for (j = 0; j < i0; j++)
525 fib->fib_masks[i].as_u32[j] = ~0;
528 fib->fib_masks[i].as_u32[i0] =
529 clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
532 if (fib->lookup_table_nbuckets == 0)
533 fib->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
535 fib->lookup_table_nbuckets = 1 << max_log2 (fib->lookup_table_nbuckets);
537 if (fib->lookup_table_size == 0)
538 fib->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
540 BV (clib_bihash_init) (&fib->ip6_lookup_table, "ip6 lookup table",
541 fib->lookup_table_nbuckets, fib->lookup_table_size);
546 ip6_sd_fib_add_del_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
547 ip_prefix_t * src_prefix, u32 table_id,
548 ip_adjacency_t * add_adj, u8 is_add)
551 ip6_add_del_route_args_t a;
552 ip_adjacency_t *dst_adjp, dst_adj;
553 ip6_address_t dst = ip_prefix_v6 (dst_prefix), src;
554 u32 dst_address_length = ip_prefix_len (dst_prefix), src_address_length = 0;
555 ip6_src_fib_t *src_fib;
559 src = ip_prefix_v6 (src_prefix);
560 src_address_length = ip_prefix_len (src_prefix);
563 memset (&src, 0, sizeof (src));
565 /* lookup dst adj and create it if it doesn't exist */
566 adj_index = ip6_get_route (lgm->im6, table_id, 0, &dst, dst_address_length);
570 /* insert dst prefix to ip6 fib, if it's not in yet */
573 /* allocate and init src ip6 fib */
574 pool_get (lgm->ip6_src_fibs, src_fib);
575 memset (src_fib, 0, sizeof (src_fib[0]));
576 ip6_src_fib_init (src_fib);
578 memset (&dst_adj, 0, sizeof (dst_adj));
580 /* reuse rewrite header to store pointer to src fib */
581 dst_adj.rewrite_header.sw_if_index = src_fib - lgm->ip6_src_fibs;
583 /* dst adj should point to lisp gpe ip lookup */
584 dst_adj.lookup_next_index = lgm->ip6_lookup_next_lgpe_ip6_lookup;
586 /* explicit_fib_index is used in IP6 FIB lookup, don't reuse it */
587 dst_adj.explicit_fib_index = ~0;
590 /* make sure we have different signatures for adj in different tables
591 * but with the same lookup_next_index and for adj in the same table
592 * but associated to different destinations */
593 dst_adj.if_address_index = table_id;
594 dst_adj.indirect.next_hop.ip6 = dst;
596 memset (&a, 0, sizeof (a));
597 a.flags = IP6_ROUTE_FLAG_TABLE_ID;
598 a.table_index_or_table_id = table_id; /* vrf */
600 a.dst_address_length = dst_address_length;
602 a.flags |= IP6_ROUTE_FLAG_ADD;
603 a.add_adj = &dst_adj;
606 ip6_add_del_route (lgm->im6, &a);
608 /* lookup dst adj to obtain the adj index */
609 adj_index = ip6_get_route (lgm->im6, table_id, 0, &dst,
612 /* make sure insertion succeeded */
615 ASSERT (adj_index != 0);
616 dst_adjp = ip_get_adjacency (lgm->lm6, adj_index);
617 ASSERT (dst_adjp->rewrite_header.sw_if_index
618 == dst_adj.rewrite_header.sw_if_index);
627 ("Trying to delete inexistent dst route for %U. Aborting",
628 format_ip_prefix, dst_prefix);
633 dst_adjp = ip_get_adjacency (lgm->lm6, adj_index);
635 /* add/del src prefix to src fib */
636 memset (&a, 0, sizeof (a));
637 a.flags = IP6_ROUTE_FLAG_TABLE_ID;
638 a.table_index_or_table_id = dst_adjp->rewrite_header.sw_if_index;
640 a.flags |= is_add ? IP6_ROUTE_FLAG_ADD : IP6_ROUTE_FLAG_DEL;
642 a.n_add_adj = is_add ? 1 : 0;
643 /* if src prefix is null, add ::0 */
644 a.dst_address_length = src_address_length;
646 ip6_sd_fib_add_del_src_route (lgm, &a);
648 /* make sure insertion succeeded */
649 if (CLIB_DEBUG && is_add)
652 ip_adjacency_t *src_adjp;
653 sai = ip6_sd_get_src_route (lgm, dst_adjp->rewrite_header.sw_if_index,
654 &src, src_address_length);
656 src_adjp = ip_get_adjacency (lgm->lm6, sai);
657 ASSERT (src_adjp->if_address_index == add_adj->if_address_index);
660 /* if a delete, check if there are elements left in the src fib */
663 src_fib = pool_elt_at_index (lgm->ip6_src_fibs,
664 dst_adjp->rewrite_header.sw_if_index);
668 /* if there's nothing left */
669 if (clib_bitmap_count_set_bits
670 (src_fib->non_empty_dst_address_length_bitmap) == 0)
672 /* remove src fib .. */
673 pool_put (lgm->ip6_src_fibs, src_fib);
675 /* .. and remove dst route */
676 memset (&a, 0, sizeof (a));
677 a.flags = IP6_ROUTE_FLAG_TABLE_ID;
678 a.table_index_or_table_id = table_id; /* vrf */
680 a.dst_address_length = dst_address_length;
682 a.flags |= IP6_ROUTE_FLAG_DEL;
684 ip6_add_del_route (lgm->im6, &a);
692 ip6_sd_fib_get_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
693 ip_prefix_t * src_prefix, u32 table_id)
696 ip6_address_t dst = ip_prefix_v6 (dst_prefix), src;
697 u32 dst_address_length = ip_prefix_len (dst_prefix), src_address_length = 0;
698 ip_adjacency_t *dst_adj;
702 src = ip_prefix_v6 (src_prefix);
703 src_address_length = ip_prefix_len (src_prefix);
706 memset (&src, 0, sizeof (src));
709 adj_index = ip6_get_route (lgm->im6, table_id, 0, &dst, dst_address_length);
713 dst_adj = ip_get_adjacency (lgm->lm6, adj_index);
714 return ip6_sd_get_src_route (lgm, dst_adj->rewrite_header.sw_if_index, &src,
719 ip_sd_fib_add_del_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
720 ip_prefix_t * src_prefix, u32 table_id,
721 ip_adjacency_t * add_adj, u8 is_add)
723 return (ip_prefix_version (dst_prefix) == IP4 ?
724 ip4_sd_fib_add_del_route : ip6_sd_fib_add_del_route) (lgm,
733 ip_sd_fib_get_route (lisp_gpe_main_t * lgm, ip_prefix_t * dst_prefix,
734 ip_prefix_t * src_prefix, u32 table_id)
736 if (ip_prefix_version (dst_prefix) == IP4)
738 u32 *adj_index = ip4_sd_fib_get_route (lgm, dst_prefix, src_prefix,
740 return (adj_index == 0) ? 0 : adj_index[0];
743 return ip6_sd_fib_get_route (lgm, dst_prefix, src_prefix, table_id);
747 ip4_src_fib_lookup_one (lisp_gpe_main_t * lgm, u32 src_fib_index0,
748 ip4_address_t * addr0, u32 * src_adj_index0)
750 ip4_fib_mtrie_leaf_t leaf0, leaf1;
751 ip4_fib_mtrie_t *mtrie0;
753 /* if default route not hit in ip4 lookup */
754 if (PREDICT_TRUE (src_fib_index0 != (u32) ~ 0))
756 mtrie0 = &vec_elt_at_index (lgm->ip4_src_fibs, src_fib_index0)->mtrie;
758 leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
759 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
760 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
761 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
762 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
764 /* Handle default route. */
765 leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY) ?
766 mtrie0->default_leaf : leaf0;
767 src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
770 src_adj_index0[0] = ~0;
774 ip4_src_fib_lookup_two (lisp_gpe_main_t * lgm, u32 src_fib_index0,
775 u32 src_fib_index1, ip4_address_t * addr0,
776 ip4_address_t * addr1, u32 * src_adj_index0,
777 u32 * src_adj_index1)
779 ip4_fib_mtrie_leaf_t leaf0, leaf1;
780 ip4_fib_mtrie_t *mtrie0, *mtrie1;
782 /* if default route not hit in ip4 lookup */
784 (src_fib_index0 != (u32) ~ 0 && src_fib_index1 != (u32) ~ 0))
786 mtrie0 = &vec_elt_at_index (lgm->ip4_src_fibs, src_fib_index0)->mtrie;
787 mtrie1 = &vec_elt_at_index (lgm->ip4_src_fibs, src_fib_index1)->mtrie;
789 leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
791 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 0);
792 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 0);
794 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 1);
795 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 1);
797 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 2);
798 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 2);
800 leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, addr0, 3);
801 leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, addr1, 3);
803 /* Handle default route. */
804 leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY) ?
805 mtrie0->default_leaf : leaf0;
806 leaf1 = (leaf1 == IP4_FIB_MTRIE_LEAF_EMPTY) ?
807 mtrie1->default_leaf : leaf1;
808 src_adj_index0[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
809 src_adj_index1[0] = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
813 ip4_src_fib_lookup_one (lgm, src_fib_index0, addr0, src_adj_index0);
814 ip4_src_fib_lookup_one (lgm, src_fib_index1, addr1, src_adj_index1);
819 lgpe_ip4_lookup (vlib_main_t * vm, vlib_node_runtime_t * node,
820 vlib_frame_t * from_frame)
822 u32 n_left_from, next_index, *from, *to_next;
823 lisp_gpe_main_t *lgm = &lisp_gpe_main;
825 from = vlib_frame_vector_args (from_frame);
826 n_left_from = from_frame->n_vectors;
828 next_index = node->cached_next_index;
830 while (n_left_from > 0)
834 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
836 while (n_left_from >= 4 && n_left_to_next >= 2)
839 vlib_buffer_t *b0, *b1;
840 ip4_header_t *ip0, *ip1;
841 u32 dst_adj_index0, src_adj_index0, src_fib_index0;
842 u32 dst_adj_index1, src_adj_index1, src_fib_index1;
843 ip_adjacency_t *dst_adj0, *src_adj0, *dst_adj1, *src_adj1;
846 next0 = next1 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
848 /* Prefetch next iteration. */
850 vlib_buffer_t *p2, *p3;
852 p2 = vlib_get_buffer (vm, from[2]);
853 p3 = vlib_get_buffer (vm, from[3]);
855 vlib_prefetch_buffer_header (p2, LOAD);
856 vlib_prefetch_buffer_header (p3, LOAD);
858 CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
859 CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
871 b0 = vlib_get_buffer (vm, bi0);
872 b1 = vlib_get_buffer (vm, bi1);
874 ip0 = vlib_buffer_get_current (b0);
875 ip1 = vlib_buffer_get_current (b1);
877 /* dst lookup was done by ip4 lookup */
878 dst_adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
879 dst_adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
881 dst_adj0 = ip_get_adjacency (lgm->lm4, dst_adj_index0);
882 dst_adj1 = ip_get_adjacency (lgm->lm4, dst_adj_index1);
884 src_fib_index0 = dst_adj0->rewrite_header.sw_if_index;
885 src_fib_index1 = dst_adj1->rewrite_header.sw_if_index;
887 ip4_src_fib_lookup_two (lgm, src_fib_index0, src_fib_index1,
888 &ip0->src_address, &ip1->src_address,
889 &src_adj_index0, &src_adj_index1);
891 /* if a source fib exists */
892 if (PREDICT_TRUE ((u32) ~ 0 != src_adj_index0
893 && (u32) ~ 0 != src_adj_index1))
895 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
896 vnet_buffer (b1)->ip.adj_index[VLIB_TX] = src_adj_index1;
898 src_adj0 = ip_get_adjacency (lgm->lm4, src_adj_index0);
899 src_adj1 = ip_get_adjacency (lgm->lm4, src_adj_index1);
901 next0 = src_adj0->explicit_fib_index;
902 next1 = src_adj1->explicit_fib_index;
904 /* prepare buffer for lisp-gpe output node */
905 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
906 src_adj0->rewrite_header.sw_if_index;
907 vnet_buffer (b1)->sw_if_index[VLIB_TX] =
908 src_adj1->rewrite_header.sw_if_index;
910 /* if multipath: saved_lookup_next_index is reused to store
911 * nb of sub-tunnels. If greater than 1, multipath is on.
912 * Note that flow hash should be 0 after ipx lookup! */
913 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
914 vnet_buffer (b0)->ip.flow_hash =
915 ip4_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
917 if (PREDICT_TRUE (src_adj1->saved_lookup_next_index > 1))
918 vnet_buffer (b1)->ip.flow_hash =
919 ip4_compute_flow_hash (ip1, IP_FLOW_HASH_DEFAULT);
923 if ((u32) ~ 0 != src_adj_index0)
925 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
926 src_adj0 = ip_get_adjacency (lgm->lm4, src_adj_index0);
927 next0 = src_adj0->explicit_fib_index;
928 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
929 src_adj0->rewrite_header.sw_if_index;
931 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
932 vnet_buffer (b0)->ip.flow_hash =
933 ip4_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
937 next0 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
940 if ((u32) ~ 0 != src_adj_index1)
942 vnet_buffer (b1)->ip.adj_index[VLIB_TX] = src_adj_index1;
943 src_adj1 = ip_get_adjacency (lgm->lm4, src_adj_index1);
944 next1 = src_adj1->explicit_fib_index;
945 vnet_buffer (b1)->sw_if_index[VLIB_TX] =
946 src_adj1->rewrite_header.sw_if_index;
947 if (PREDICT_TRUE (src_adj1->saved_lookup_next_index > 1))
948 vnet_buffer (b1)->ip.flow_hash =
949 ip4_compute_flow_hash (ip1, IP_FLOW_HASH_DEFAULT);
953 next1 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
957 /* mark the packets for CP lookup if needed */
958 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next0))
959 vnet_buffer (b0)->lisp.overlay_afi = LISP_AFI_IP;
960 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next1))
961 vnet_buffer (b1)->lisp.overlay_afi = LISP_AFI_IP;
963 vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
964 n_left_to_next, bi0, bi1, next0,
968 while (n_left_from > 0 && n_left_to_next > 0)
972 u32 bi0, dst_adj_index0, src_adj_index0, src_fib_index0;
973 u32 next0 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
974 ip_adjacency_t *dst_adj0, *src_adj0;
983 b0 = vlib_get_buffer (vm, bi0);
984 ip0 = vlib_buffer_get_current (b0);
986 /* dst lookup was done by ip4 lookup */
987 dst_adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
988 dst_adj0 = ip_get_adjacency (lgm->lm4, dst_adj_index0);
989 src_fib_index0 = dst_adj0->rewrite_header.sw_if_index;
992 ip4_src_fib_lookup_one (lgm, src_fib_index0, &ip0->src_address,
995 /* if a source fib exists */
996 if (PREDICT_TRUE ((u32) ~ 0 != src_adj_index0))
998 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
999 src_adj0 = ip_get_adjacency (lgm->lm4, src_adj_index0);
1000 next0 = src_adj0->explicit_fib_index;
1002 /* prepare packet for lisp-gpe output node */
1003 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
1004 src_adj0->rewrite_header.sw_if_index;
1006 /* if multipath: saved_lookup_next_index is reused to store
1007 * nb of sub-tunnels. If greater than 1, multipath is on */
1008 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
1009 vnet_buffer (b0)->ip.flow_hash =
1010 ip4_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
1014 next0 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
1017 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next0))
1018 vnet_buffer (b0)->lisp.overlay_afi = LISP_AFI_IP;
1020 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1021 n_left_to_next, bi0, next0);
1023 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1025 return from_frame->n_vectors;
1029 VLIB_REGISTER_NODE (lgpe_ip4_lookup_node) = {
1030 .function = lgpe_ip4_lookup,
1031 .name = "lgpe-ip4-lookup",
1032 .vector_size = sizeof (u32),
1034 .type = VLIB_NODE_TYPE_INTERNAL,
1036 .n_next_nodes = LGPE_IP4_LOOKUP_N_NEXT,
1038 #define _(sym,str) [LGPE_IP4_LOOKUP_NEXT_##sym] = str,
1039 foreach_lgpe_ip4_lookup_next
1046 ip6_src_fib_lookup (lisp_gpe_main_t * lgm, u32 src_fib_index,
1047 ip6_address_t * src)
1051 BVT (clib_bihash_kv) kv, value;
1052 ip6_src_fib_t *fib = pool_elt_at_index (lgm->ip6_src_fibs, src_fib_index);
1054 len = vec_len (fib->prefix_lengths_in_search_order);
1056 for (i = 0; i < len; i++)
1058 int dst_address_length = fib->prefix_lengths_in_search_order[i];
1059 ip6_address_t *mask;
1061 ASSERT (dst_address_length >= 0 && dst_address_length <= 128);
1063 mask = &fib->fib_masks[dst_address_length];
1065 kv.key[0] = src->as_u64[0] & mask->as_u64[0];
1066 kv.key[1] = src->as_u64[1] & mask->as_u64[1];
1067 kv.key[2] = dst_address_length;
1070 BV (clib_bihash_search_inline_2) (&fib->ip6_lookup_table, &kv,
1080 ip6_src_fib_lookup_one (lisp_gpe_main_t * lgm, u32 src_fib_index0,
1081 ip6_address_t * addr0, u32 * src_adj_index0)
1083 /* if default route not hit in ip6 lookup */
1084 if (PREDICT_TRUE (src_fib_index0 != (u32) ~ 0))
1085 src_adj_index0[0] = ip6_src_fib_lookup (lgm, src_fib_index0, addr0);
1087 src_adj_index0[0] = ~0;
1091 ip6_src_fib_lookup_two (lisp_gpe_main_t * lgm, u32 src_fib_index0,
1092 u32 src_fib_index1, ip6_address_t * addr0,
1093 ip6_address_t * addr1, u32 * src_adj_index0,
1094 u32 * src_adj_index1)
1096 /* if default route not hit in ip6 lookup */
1098 (src_fib_index0 != (u32) ~ 0 && src_fib_index1 != (u32) ~ 0))
1100 src_adj_index0[0] = ip6_src_fib_lookup (lgm, src_fib_index0, addr0);
1101 src_adj_index1[0] = ip6_src_fib_lookup (lgm, src_fib_index1, addr1);
1105 ip6_src_fib_lookup_one (lgm, src_fib_index0, addr0, src_adj_index0);
1106 ip6_src_fib_lookup_one (lgm, src_fib_index1, addr1, src_adj_index1);
1111 lgpe_ip6_lookup (vlib_main_t * vm, vlib_node_runtime_t * node,
1112 vlib_frame_t * from_frame)
1114 u32 n_left_from, next_index, *from, *to_next;
1115 lisp_gpe_main_t *lgm = &lisp_gpe_main;
1117 from = vlib_frame_vector_args (from_frame);
1118 n_left_from = from_frame->n_vectors;
1120 next_index = node->cached_next_index;
1122 while (n_left_from > 0)
1126 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1128 while (n_left_from >= 4 && n_left_to_next >= 2)
1131 vlib_buffer_t *b0, *b1;
1132 ip6_header_t *ip0, *ip1;
1133 u32 dst_adj_index0, src_adj_index0, src_fib_index0, dst_adj_index1,
1134 src_adj_index1, src_fib_index1;
1135 ip_adjacency_t *dst_adj0, *src_adj0, *dst_adj1, *src_adj1;
1138 next0 = next1 = LGPE_IP6_LOOKUP_NEXT_LISP_CP_LOOKUP;
1140 /* Prefetch next iteration. */
1142 vlib_buffer_t *p2, *p3;
1144 p2 = vlib_get_buffer (vm, from[2]);
1145 p3 = vlib_get_buffer (vm, from[3]);
1147 vlib_prefetch_buffer_header (p2, LOAD);
1148 vlib_prefetch_buffer_header (p3, LOAD);
1150 CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
1151 CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
1160 n_left_to_next -= 2;
1163 b0 = vlib_get_buffer (vm, bi0);
1164 b1 = vlib_get_buffer (vm, bi1);
1166 ip0 = vlib_buffer_get_current (b0);
1167 ip1 = vlib_buffer_get_current (b1);
1169 /* dst lookup was done by ip6 lookup */
1170 dst_adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1171 dst_adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
1173 dst_adj0 = ip_get_adjacency (lgm->lm6, dst_adj_index0);
1174 dst_adj1 = ip_get_adjacency (lgm->lm6, dst_adj_index1);
1176 src_fib_index0 = dst_adj0->rewrite_header.sw_if_index;
1177 src_fib_index1 = dst_adj1->rewrite_header.sw_if_index;
1179 ip6_src_fib_lookup_two (lgm, src_fib_index0, src_fib_index1,
1180 &ip0->src_address, &ip1->src_address,
1181 &src_adj_index0, &src_adj_index1);
1183 /* if a source fib exists */
1184 if (PREDICT_TRUE ((u32) ~ 0 != src_adj_index0
1185 && (u32) ~ 0 != src_adj_index1))
1187 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
1188 vnet_buffer (b1)->ip.adj_index[VLIB_TX] = src_adj_index1;
1190 src_adj0 = ip_get_adjacency (lgm->lm6, src_adj_index0);
1191 src_adj1 = ip_get_adjacency (lgm->lm6, src_adj_index1);
1193 next0 = src_adj0->explicit_fib_index;
1194 next1 = src_adj1->explicit_fib_index;
1196 /* prepare buffer for lisp-gpe output node */
1197 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
1198 src_adj0->rewrite_header.sw_if_index;
1199 vnet_buffer (b1)->sw_if_index[VLIB_TX] =
1200 src_adj1->rewrite_header.sw_if_index;
1202 /* if multipath: saved_lookup_next_index is reused to store
1203 * nb of sub-tunnels. If greater than 1, multipath is on.
1204 * Note that flow hash should be 0 after ipx lookup! */
1205 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
1206 vnet_buffer (b0)->ip.flow_hash =
1207 ip6_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
1209 if (PREDICT_TRUE (src_adj1->saved_lookup_next_index > 1))
1210 vnet_buffer (b1)->ip.flow_hash =
1211 ip6_compute_flow_hash (ip1, IP_FLOW_HASH_DEFAULT);
1215 if (src_adj_index0 != (u32) ~ 0)
1217 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
1218 src_adj0 = ip_get_adjacency (lgm->lm6, src_adj_index0);
1219 next0 = src_adj0->explicit_fib_index;
1220 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
1221 src_adj0->rewrite_header.sw_if_index;
1223 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
1224 vnet_buffer (b0)->ip.flow_hash =
1225 ip6_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
1229 next0 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
1232 if (src_adj_index1 != (u32) ~ 0)
1234 vnet_buffer (b1)->ip.adj_index[VLIB_TX] = src_adj_index1;
1235 src_adj1 = ip_get_adjacency (lgm->lm6, src_adj_index1);
1236 next1 = src_adj1->explicit_fib_index;
1237 vnet_buffer (b1)->sw_if_index[VLIB_TX] =
1238 src_adj1->rewrite_header.sw_if_index;
1240 if (PREDICT_TRUE (src_adj1->saved_lookup_next_index > 1))
1241 vnet_buffer (b1)->ip.flow_hash =
1242 ip6_compute_flow_hash (ip1, IP_FLOW_HASH_DEFAULT);
1246 next1 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
1250 /* mark the packets for CP lookup if needed */
1251 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next0))
1252 vnet_buffer (b0)->lisp.overlay_afi = LISP_AFI_IP;
1253 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next1))
1254 vnet_buffer (b1)->lisp.overlay_afi = LISP_AFI_IP;
1256 vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1257 n_left_to_next, bi0, bi1, next0,
1261 while (n_left_from > 0 && n_left_to_next > 0)
1265 u32 bi0, dst_adj_index0, src_adj_index0, src_fib_index0;
1266 u32 next0 = LGPE_IP6_LOOKUP_NEXT_LISP_CP_LOOKUP;
1267 ip_adjacency_t *dst_adj0, *src_adj0;
1274 n_left_to_next -= 1;
1276 b0 = vlib_get_buffer (vm, bi0);
1277 ip0 = vlib_buffer_get_current (b0);
1279 /* dst lookup was done by ip6 lookup */
1280 dst_adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
1281 dst_adj0 = ip_get_adjacency (lgm->lm6, dst_adj_index0);
1282 src_fib_index0 = dst_adj0->rewrite_header.sw_if_index;
1285 ip6_src_fib_lookup_one (lgm, src_fib_index0, &ip0->src_address,
1288 /* if a source fib exists */
1289 if (PREDICT_TRUE (src_adj_index0 != (u32) ~ 0))
1291 vnet_buffer (b0)->ip.adj_index[VLIB_TX] = src_adj_index0;
1292 src_adj0 = ip_get_adjacency (lgm->lm6, src_adj_index0);
1293 next0 = src_adj0->explicit_fib_index;
1295 /* prepare packet for lisp-gpe output node */
1296 vnet_buffer (b0)->sw_if_index[VLIB_TX] =
1297 src_adj0->rewrite_header.sw_if_index;
1299 /* if multipath: saved_lookup_next_index is reused to store
1300 * nb of sub-tunnels. If greater than 1, multipath is on */
1301 if (PREDICT_TRUE (src_adj0->saved_lookup_next_index > 1))
1302 vnet_buffer (b0)->ip.flow_hash =
1303 ip6_compute_flow_hash (ip0, IP_FLOW_HASH_DEFAULT);
1307 next0 = LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP;
1310 /* mark the packets for CP lookup if needed */
1311 if (PREDICT_FALSE (LGPE_IP4_LOOKUP_NEXT_LISP_CP_LOOKUP == next0))
1312 vnet_buffer (b0)->lisp.overlay_afi = LISP_AFI_IP;
1314 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1315 n_left_to_next, bi0, next0);
1317 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1319 return from_frame->n_vectors;
1323 VLIB_REGISTER_NODE (lgpe_ip6_lookup_node) = {
1324 .function = lgpe_ip6_lookup,
1325 .name = "lgpe-ip6-lookup",
1326 .vector_size = sizeof (u32),
1328 .type = VLIB_NODE_TYPE_INTERNAL,
1330 .n_next_nodes = LGPE_IP6_LOOKUP_N_NEXT,
1332 #define _(sym,str) [LGPE_IP6_LOOKUP_NEXT_##sym] = str,
1333 foreach_lgpe_ip6_lookup_next
1340 * fd.io coding-style-patch-verification: ON
1343 * eval: (c-set-style "gnu")