vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45
  46 #include <vppinfra/bihash_template.c>
  47
  48 static void compute_prefix_lengths_in_search_order (ip6_main_t * im)
  49 {
  50   int i;
  51   vec_reset_length (im->prefix_lengths_in_search_order);
  52   /* Note: bitmap reversed so this is in fact a longest prefix match */
  53   clib_bitmap_foreach (i, im->non_empty_dst_address_length_bitmap,
  54   ({
  55     int dst_address_length = 128 - i;
  56     vec_add1 (im->prefix_lengths_in_search_order, dst_address_length);
  57   }));
  58 }
  59
  60 u32
  61 ip6_fib_lookup_with_table (ip6_main_t * im, u32 fib_index, ip6_address_t * dst)
  62 {
  63   ip_lookup_main_t * lm = &im->lookup_main;
  64   int i, len;
  65   int rv;
  66   BVT(clib_bihash_kv) kv, value;
  67   u64 fib;
  68
  69   len = vec_len (im->prefix_lengths_in_search_order);
  70
  71   kv.key[0] = dst->as_u64[0];
  72   kv.key[1] = dst->as_u64[1];
  73   fib = ((u64)((fib_index))<<32);
  74
  75   for (i = 0; i < len; i++)
  76     {
  77       int dst_address_length = im->prefix_lengths_in_search_order[i];
  78       ip6_address_t * mask = &im->fib_masks[dst_address_length];
  79
  80       ASSERT(dst_address_length >= 0 && dst_address_length <= 128);
  81       //As lengths are decreasing, masks are increasingly specific.
  82       kv.key[0] &= mask->as_u64[0];
  83       kv.key[1] &= mask->as_u64[1];
  84       kv.key[2] = fib | dst_address_length;
  85
  86       rv = BV(clib_bihash_search_inline_2)(&im->ip6_lookup_table, &kv, &value);
  87       if (rv == 0)
  88         return value.value;
  89     }
  90
  91   return lm->miss_adj_index;
  92 }
  93
  94 u32 ip6_fib_lookup (ip6_main_t * im, u32 sw_if_index, ip6_address_t * dst)
  95 {
  96     u32 fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
  97     return ip6_fib_lookup_with_table (im, fib_index, dst);
  98 }
  99
 100 void
 101 vnet_ip6_fib_init (ip6_main_t * im, u32 fib_index)
 102 {
 103   ip_lookup_main_t * lm = &im->lookup_main;
 104   ip6_add_del_route_args_t a;
 105   ip_adjacency_t * adj;
 106
 107   memset(&a, 0x0, sizeof(ip6_add_del_route_args_t));
 108
 109   a.table_index_or_table_id = fib_index;
 110   a.flags = (IP6_ROUTE_FLAG_ADD
 111              | IP6_ROUTE_FLAG_FIB_INDEX
 112              | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 113              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
 114
 115   /* Add ff02::1:ff00:0/104 via local route for all tables.
 116      This is required for neighbor discovery to work. */
 117   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 118                           &a.adj_index);
 119   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 120   adj->if_address_index = ~0;
 121   adj->rewrite_header.data_bytes = 0;
 122
 123   ip6_set_solicited_node_multicast_address (&a.dst_address, 0);
 124
 125   a.dst_address_length = 104;
 126   ip6_add_del_route (im, &a);
 127
 128   /* Add all-routers multicast address via local route for all tables */
 129   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 130                           &a.adj_index);
 131   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 132   adj->if_address_index = ~0;
 133   adj->rewrite_header.data_bytes = 0;
 134
 135   ip6_set_reserved_multicast_address (&a.dst_address,
 136                                       IP6_MULTICAST_SCOPE_link_local,
 137                                       IP6_MULTICAST_GROUP_ID_all_routers);
 138
 139   a.dst_address_length = 128;
 140   ip6_add_del_route (im, &a);
 141
 142   /* Add all-nodes multicast address via local route for all tables */
 143   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 144                           &a.adj_index);
 145   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 146   adj->if_address_index = ~0;
 147   adj->rewrite_header.data_bytes = 0;
 148
 149   ip6_set_reserved_multicast_address (&a.dst_address,
 150                                       IP6_MULTICAST_SCOPE_link_local,
 151                                       IP6_MULTICAST_GROUP_ID_all_hosts);
 152
 153   a.dst_address_length = 128;
 154   ip6_add_del_route (im, &a);
 155
 156   /* Add all-mldv2  multicast address via local route for all tables */
 157   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 158                           &a.adj_index);
 159   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 160   adj->if_address_index = ~0;
 161   adj->rewrite_header.data_bytes = 0;
 162
 163   ip6_set_reserved_multicast_address (&a.dst_address,
 164                                       IP6_MULTICAST_SCOPE_link_local,
 165                                       IP6_MULTICAST_GROUP_ID_mldv2_routers);
 166
 167   a.dst_address_length = 128;
 168   ip6_add_del_route (im, &a);
 169 }
 170
 171 static ip6_fib_t *
 172 create_fib_with_table_id (ip6_main_t * im, u32 table_id)
 173 {
 174   ip6_fib_t * fib;
 175   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
 176   vec_add2 (im->fibs, fib, 1);
 177   fib->table_id = table_id;
 178   fib->index = fib - im->fibs;
 179   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
 180   vnet_ip6_fib_init (im, fib->index);
 181   return fib;
 182 }
 183
 184 ip6_fib_t *
 185 find_ip6_fib_by_table_index_or_id (ip6_main_t * im, u32 table_index_or_id, u32 flags)
 186 {
 187   uword * p, fib_index;
 188
 189   fib_index = table_index_or_id;
 190   if (! (flags & IP6_ROUTE_FLAG_FIB_INDEX))
 191     {
 192       if (table_index_or_id == ~0) {
 193         table_index_or_id = 0;
 194         while (hash_get (im->fib_index_by_table_id, table_index_or_id)) {
 195           table_index_or_id++;
 196         }
 197         return create_fib_with_table_id (im, table_index_or_id);
 198       }
 199
 200       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 201       if (! p)
 202         return create_fib_with_table_id (im, table_index_or_id);
 203       fib_index = p[0];
 204     }
 205   return vec_elt_at_index (im->fibs, fib_index);
 206 }
 207
 208 void ip6_add_del_route (ip6_main_t * im, ip6_add_del_route_args_t * a)
 209 {
 210   ip_lookup_main_t * lm = &im->lookup_main;
 211   ip6_fib_t * fib;
 212   ip6_address_t dst_address;
 213   u32 dst_address_length, adj_index;
 214   uword is_del;
 215   u32 old_adj_index = ~0;
 216   BVT(clib_bihash_kv) kv, value;
 217
 218   vlib_smp_unsafe_warning();
 219
 220   is_del = (a->flags & IP6_ROUTE_FLAG_DEL) != 0;
 221
 222   /* Either create new adjacency or use given one depending on arguments. */
 223   if (a->n_add_adj > 0)
 224     {
 225       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 226       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 227     }
 228   else
 229     adj_index = a->adj_index;
 230
 231   dst_address = a->dst_address;
 232   dst_address_length = a->dst_address_length;
 233   fib = find_ip6_fib_by_table_index_or_id (im, a->table_index_or_table_id,
 234                                            a->flags);
 235
 236   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 237   ip6_address_mask (&dst_address, &im->fib_masks[dst_address_length]);
 238
 239   /* refcount accounting */
 240   if (is_del)
 241     {
 242       ASSERT (im->dst_address_length_refcounts[dst_address_length] > 0);
 243       if (--im->dst_address_length_refcounts[dst_address_length] == 0)
 244         {
 245           im->non_empty_dst_address_length_bitmap =
 246             clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 247                              128 - dst_address_length, 0);
 248           compute_prefix_lengths_in_search_order (im);
 249         }
 250     }
 251   else
 252     {
 253       im->dst_address_length_refcounts[dst_address_length]++;
 254
 255       im->non_empty_dst_address_length_bitmap =
 256         clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 257                              128 - dst_address_length, 1);
 258       compute_prefix_lengths_in_search_order (im);
 259     }
 260
 261   kv.key[0] = dst_address.as_u64[0];
 262   kv.key[1] = dst_address.as_u64[1];
 263   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 264
 265   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 266     old_adj_index = value.value;
 267
 268   if (is_del)
 269     BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 0 /* is_add */);
 270   else
 271     {
 272       /* Make sure adj index is valid. */
 273       if (CLIB_DEBUG > 0)
 274         (void) ip_get_adjacency (lm, adj_index);
 275
 276       kv.value = adj_index;
 277
 278       BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 1 /* is_add */);
 279     }
 280
 281   /* Avoid spurious reference count increments */
 282   if (old_adj_index == adj_index
 283       && adj_index != ~0
 284       && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 285     {
 286       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 287       if (adj->share_count > 0)
 288         adj->share_count --;
 289     }
 290
 291   /* Delete old adjacency index if present and changed. */
 292   {
 293     if (! (a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 294         && old_adj_index != ~0
 295         && old_adj_index != adj_index)
 296       ip_del_adjacency (lm, old_adj_index);
 297   }
 298 }
 299
 300 u32
 301 ip6_route_get_next_hop_adj (ip6_main_t * im,
 302                             u32 fib_index,
 303                             ip6_address_t *next_hop,
 304                             u32 next_hop_sw_if_index,
 305                             u32 explicit_fib_index)
 306 {
 307   ip_lookup_main_t * lm = &im->lookup_main;
 308   vnet_main_t * vnm = vnet_get_main();
 309   int is_interface_next_hop;
 310   uword * nh_result;
 311   u32 nh_adj_index;
 312   ip6_fib_t * fib;
 313
 314   fib = vec_elt_at_index (im->fibs, fib_index);
 315
 316   is_interface_next_hop = ip6_address_is_zero (next_hop);
 317
 318   if (is_interface_next_hop)
 319     {
 320       nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index,
 321                             next_hop_sw_if_index);
 322       if (nh_result)
 323           nh_adj_index = *nh_result;
 324       else
 325         {
 326           ip_adjacency_t * adj;
 327           adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 328                                   &nh_adj_index);
 329           ip6_adjacency_set_interface_route (vnm, adj,
 330                                              next_hop_sw_if_index, ~0);
 331           ip_call_add_del_adjacency_callbacks
 332               (lm, next_hop_sw_if_index, /* is_del */ 0);
 333           hash_set (im->interface_route_adj_index_by_sw_if_index,
 334                     next_hop_sw_if_index, nh_adj_index);
 335         }
 336     }
 337   else if (next_hop_sw_if_index == ~0)
 338     {
 339       /* next-hop is recursive. we always need a indirect adj
 340        * for recursive paths. Any LPM we perform now will give
 341        * us a valid adj, but without tracking the next-hop we
 342        * have no way to keep it valid.
 343        */
 344       ip_adjacency_t add_adj;
 345       memset (&add_adj, 0, sizeof(add_adj));
 346       add_adj.n_adj = 1;
 347       add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 348       add_adj.indirect.next_hop.ip6.as_u64[0] = next_hop->as_u64[0];
 349       add_adj.indirect.next_hop.ip6.as_u64[1] = next_hop->as_u64[1];
 350       add_adj.explicit_fib_index = explicit_fib_index;
 351       ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 352     }
 353   else
 354     {
 355       BVT(clib_bihash_kv) kv, value;
 356
 357       /* Look for the interface /128 route */
 358       kv.key[0] = next_hop->as_u64[0];
 359       kv.key[1] = next_hop->as_u64[1];
 360       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 361 after_nd:
 362       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 363         {
 364           ip_adjacency_t * adj;
 365           nh_adj_index = ip6_fib_lookup_with_table (im, fib_index, next_hop);
 366           adj = ip_get_adjacency (lm, nh_adj_index);
 367           /* if ND interface adjacencty is present, we need to
 368            install ND adjaceny for specific next hop */
 369           if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 370               adj->arp.next_hop.ip6.as_u64[0] == 0 &&
 371               adj->arp.next_hop.ip6.as_u64[1] == 0)
 372             {
 373               nh_adj_index = vnet_ip6_neighbor_glean_add(fib_index, next_hop);
 374             }
 375           else if (next_hop->as_u8[0] == 0xfe)
 376             {
 377               //Next hop is link-local. No indirect in this case.
 378               //Let's add it as a possible neighbor on this interface
 379               ip6_address_t null_addr= {};
 380               ip6_add_del_route_next_hop (im, IP6_ROUTE_FLAG_ADD,
 381                                           next_hop, 128,
 382                                           &null_addr, next_hop_sw_if_index,
 383                                           1, ~0, fib_index);
 384               goto after_nd;
 385             }
 386         }
 387       else
 388         {
 389           nh_adj_index = value.value;
 390         }
 391     }
 392
 393   return (nh_adj_index);
 394 }
 395
 396 void
 397 ip6_add_del_route_next_hop (ip6_main_t * im,
 398                             u32 flags,
 399                             ip6_address_t * dst_address,
 400                             u32 dst_address_length,
 401                             ip6_address_t * next_hop,
 402                             u32 next_hop_sw_if_index,
 403                             u32 next_hop_weight, u32 adj_index,
 404                             u32 explicit_fib_index)
 405 {
 406   vnet_main_t * vnm = vnet_get_main();
 407   ip_lookup_main_t * lm = &im->lookup_main;
 408   u32 fib_index;
 409   ip6_fib_t * fib;
 410   ip6_address_t masked_dst_address;
 411   u32 old_mp_adj_index, new_mp_adj_index;
 412   u32 dst_adj_index, nh_adj_index;
 413   int rv;
 414   ip_adjacency_t * dst_adj;
 415   ip_multipath_adjacency_t * old_mp, * new_mp;
 416   int is_del = (flags & IP6_ROUTE_FLAG_DEL) != 0;
 417   clib_error_t * error = 0;
 418   BVT(clib_bihash_kv) kv, value;
 419
 420   vlib_smp_unsafe_warning();
 421
 422   if (explicit_fib_index == (u32)~0)
 423     fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 424   else
 425     fib_index = explicit_fib_index;
 426
 427   fib = vec_elt_at_index (im->fibs, fib_index);
 428
 429   /* Lookup next hop to be added or deleted. */
 430   if (adj_index == (u32)~0)
 431     {
 432       nh_adj_index = ip6_route_get_next_hop_adj(im, fib_index,
 433                                                 next_hop,
 434                                                 next_hop_sw_if_index,
 435                                                 explicit_fib_index);
 436     }
 437   else
 438     {
 439       /* Look for the interface /128 route */
 440       kv.key[0] = next_hop->as_u64[0];
 441       kv.key[1] = next_hop->as_u64[1];
 442       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 443
 444       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 445         {
 446           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 447           error = clib_error_return (0, "next-hop %U/128 not in FIB",
 448                                      format_ip6_address, next_hop);
 449           goto done;
 450         }
 451
 452       nh_adj_index = value.value;
 453     }
 454
 455   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 456   masked_dst_address = dst_address[0];
 457   ip6_address_mask (&masked_dst_address, &im->fib_masks[dst_address_length]);
 458
 459   kv.key[0] = masked_dst_address.as_u64[0];
 460   kv.key[1] = masked_dst_address.as_u64[1];
 461   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 462
 463   rv = BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value);
 464
 465   if (rv == 0)
 466     {
 467       dst_adj_index = value.value;
 468       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 469     }
 470   else
 471     {
 472       /* For deletes destination must be known. */
 473       if (is_del)
 474         {
 475           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 476           error = clib_error_return (0, "unknown destination %U/%d",
 477                                      format_ip6_address, dst_address,
 478                                      dst_address_length);
 479           goto done;
 480         }
 481
 482       dst_adj_index = ~0;
 483       dst_adj = 0;
 484     }
 485
 486   /* Ignore adds of X/128 with next hop of X. */
 487   if (! is_del
 488       && dst_address_length == 128
 489       && ip6_address_is_equal (dst_address, next_hop))
 490     {
 491       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 492       error = clib_error_return (0, "prefix matches next hop %U/%d",
 493                                  format_ip6_address, dst_address,
 494                                  dst_address_length);
 495       goto done;
 496     }
 497
 498   /* Destination is not known and default weight is set so add route
 499      to existing non-multipath adjacency */
 500   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 501   {
 502     /* create / delete additional mapping of existing adjacency */
 503     ip6_add_del_route_args_t a;
 504
 505     a.table_index_or_table_id = fib_index;
 506     a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 507         | IP6_ROUTE_FLAG_FIB_INDEX
 508         | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 509         | (flags & (IP6_ROUTE_FLAG_NO_REDISTRIBUTE
 510             | IP6_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 511     a.dst_address = dst_address[0];
 512     a.dst_address_length = dst_address_length;
 513     a.adj_index = nh_adj_index;
 514     a.add_adj = 0;
 515     a.n_add_adj = 0;
 516
 517     ip6_add_del_route (im, &a);
 518     goto done;
 519   }
 520
 521   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 522
 523   if (! ip_multipath_adjacency_add_del_next_hop
 524       (lm, is_del,
 525        dst_adj ? dst_adj->heap_handle : ~0,
 526        nh_adj_index,
 527        next_hop_weight,
 528        &new_mp_adj_index))
 529     {
 530       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 531       error = clib_error_return
 532         (0, "requested deleting next-hop %U not found in multi-path",
 533          format_ip6_address, next_hop);
 534       goto done;
 535     }
 536
 537   old_mp = new_mp = 0;
 538   if (old_mp_adj_index != ~0)
 539     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 540   if (new_mp_adj_index != ~0)
 541     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 542
 543   if (old_mp != new_mp)
 544     {
 545       ip6_add_del_route_args_t a;
 546       ip_adjacency_t * adj;
 547
 548       a.table_index_or_table_id = fib_index;
 549       a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 550                  | IP6_ROUTE_FLAG_FIB_INDEX
 551                  | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 552                  | (flags & IP6_ROUTE_FLAG_NO_REDISTRIBUTE));
 553       a.dst_address = dst_address[0];
 554       a.dst_address_length = dst_address_length;
 555       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 556       a.add_adj = 0;
 557       a.n_add_adj = 0;
 558
 559       ip6_add_del_route (im, &a);
 560
 561       adj = ip_get_adjacency (lm, new_mp ? new_mp->adj_index : dst_adj_index);
 562       if (adj->n_adj == 1)
 563         adj->share_count += is_del ? -1 : 1;
 564     }
 565
 566  done:
 567   if (error)
 568     clib_error_report (error);
 569 }
 570
 571 u32
 572 ip6_get_route (ip6_main_t * im,
 573                u32 table_index_or_table_id,
 574                u32 flags,
 575                ip6_address_t * address,
 576                u32 address_length)
 577 {
 578   ip6_fib_t * fib = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 579   ip6_address_t masked_address;
 580   BVT(clib_bihash_kv) kv, value;
 581
 582   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 583   clib_memcpy (&masked_address, address, sizeof (masked_address));
 584   ip6_address_mask (&masked_address, &im->fib_masks[address_length]);
 585
 586   kv.key[0] = masked_address.as_u64[0];
 587   kv.key[1] = masked_address.as_u64[1];
 588   kv.key[2] = ((u64)((fib - im->fibs))<<32) | address_length;
 589
 590   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 591     return (value.value);
 592   return 0;
 593 }
 594
 595 void
 596 ip6_foreach_matching_route (ip6_main_t * im,
 597                             u32 table_index_or_table_id,
 598                             u32 flags,
 599                             ip6_address_t * dst_address,
 600                             u32 address_length,
 601                             ip6_address_t ** results,
 602                             u8 ** result_lengths)
 603 {
 604   ip6_fib_t * fib =
 605     find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 606   BVT(clib_bihash) * h = &im->ip6_lookup_table;
 607   BVT(clib_bihash_value) * v;
 608   clib_bihash_bucket_t * b;
 609   int i, j, k;
 610
 611   if (*results)
 612     _vec_len (*results) = 0;
 613   if (*result_lengths)
 614     _vec_len (*result_lengths) = 0;
 615
 616   /* Walk the table looking for routes which match the supplied address */
 617   for (i = 0; i < h->nbuckets; i++)
 618     {
 619       b = &h->buckets [i];
 620       if (b->offset == 0)
 621           continue;
 622
 623       v = BV(clib_bihash_get_value) (h, b->offset);
 624       for (j = 0; j < (1<<b->log2_pages); j++)
 625         {
 626           for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
 627             {
 628               if (BV(clib_bihash_is_free)(&v->kvp[k]))
 629                 continue;
 630
 631               if ((v->kvp[k].key[2]
 632                    == (((u64)((fib - im->fibs))<<32) | address_length))
 633                   && ip6_destination_matches_route
 634                   (im, dst_address, (ip6_address_t *) &v->kvp[k],
 635                    address_length))
 636                 {
 637                   ip6_address_t * a;
 638
 639                   a = (ip6_address_t *)(&v->kvp[k]);
 640
 641                   vec_add1 (*results, a[0]);
 642                   vec_add1 (*result_lengths, address_length);
 643                 }
 644             }
 645           v++;
 646         }
 647     }
 648 }
 649
 650 void ip6_maybe_remap_adjacencies (ip6_main_t * im,
 651                                   u32 table_index_or_table_id,
 652                                   u32 flags)
 653 {
 654 #if SOONE
 655   ip6_fib_t * fib
 656     = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 657 #endif
 658   ip_lookup_main_t * lm = &im->lookup_main;
 659
 660   if (lm->n_adjacency_remaps == 0)
 661     return;
 662
 663   clib_warning ("unimplemented, please report to vpp-dev@cisco.com");
 664
 665   /* All remaps have been performed. */
 666   lm->n_adjacency_remaps = 0;
 667 }
 668
 669 void ip6_delete_matching_routes (ip6_main_t * im,
 670                                  u32 table_index_or_table_id,
 671                                  u32 flags,
 672                                  ip6_address_t * address,
 673                                  u32 address_length)
 674 {
 675   /* $$$$ static may be OK - this should happen only on thread 0 */
 676   static ip6_address_t * matching_addresses;
 677   static u8 * matching_address_lengths;
 678   u32 l, i;
 679   ip6_add_del_route_args_t a;
 680
 681   vlib_smp_unsafe_warning();
 682
 683   a.flags = IP6_ROUTE_FLAG_DEL | IP6_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 684   a.table_index_or_table_id = table_index_or_table_id;
 685   a.adj_index = ~0;
 686   a.add_adj = 0;
 687   a.n_add_adj = 0;
 688
 689   for (l = address_length + 1; l <= 128; l++)
 690     {
 691       ip6_foreach_matching_route (im, table_index_or_table_id, flags,
 692                                   address,
 693                                   l,
 694                                   &matching_addresses,
 695                                   &matching_address_lengths);
 696       for (i = 0; i < vec_len (matching_addresses); i++)
 697         {
 698           a.dst_address = matching_addresses[i];
 699           a.dst_address_length = matching_address_lengths[i];
 700           ip6_add_del_route (im, &a);
 701         }
 702     }
 703
 704   ip6_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 705 }
 706
 707 void
 708 ip6_forward_next_trace (vlib_main_t * vm,
 709                         vlib_node_runtime_t * node,
 710                         vlib_frame_t * frame,
 711                         vlib_rx_or_tx_t which_adj_index);
 712
 713 always_inline uword
 714 ip6_lookup_inline (vlib_main_t * vm,
 715                    vlib_node_runtime_t * node,
 716                    vlib_frame_t * frame,
 717                    int is_indirect)
 718 {
 719   ip6_main_t * im = &ip6_main;
 720   ip_lookup_main_t * lm = &im->lookup_main;
 721   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 722   u32 n_left_from, n_left_to_next, * from, * to_next;
 723   ip_lookup_next_t next;
 724   u32 cpu_index = os_get_cpu_number();
 725
 726   from = vlib_frame_vector_args (frame);
 727   n_left_from = frame->n_vectors;
 728   next = node->cached_next_index;
 729
 730   while (n_left_from > 0)
 731     {
 732       vlib_get_next_frame (vm, node, next,
 733                            to_next, n_left_to_next);
 734
 735       while (n_left_from >= 4 && n_left_to_next >= 2)
 736         {
 737           vlib_buffer_t * p0, * p1;
 738           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
 739           ip_lookup_next_t next0, next1;
 740           ip6_header_t * ip0, * ip1;
 741           ip_adjacency_t * adj0, * adj1;
 742           ip6_address_t * dst_addr0, * dst_addr1;
 743           u32 fib_index0, fib_index1;
 744           u32 flow_hash_config0, flow_hash_config1;
 745
 746           /* Prefetch next iteration. */
 747           {
 748             vlib_buffer_t * p2, * p3;
 749
 750             p2 = vlib_get_buffer (vm, from[2]);
 751             p3 = vlib_get_buffer (vm, from[3]);
 752
 753             vlib_prefetch_buffer_header (p2, LOAD);
 754             vlib_prefetch_buffer_header (p3, LOAD);
 755             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 756             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 757           }
 758
 759           pi0 = to_next[0] = from[0];
 760           pi1 = to_next[1] = from[1];
 761
 762           p0 = vlib_get_buffer (vm, pi0);
 763           p1 = vlib_get_buffer (vm, pi1);
 764
 765           ip0 = vlib_buffer_get_current (p0);
 766           ip1 = vlib_buffer_get_current (p1);
 767
 768           if (is_indirect)
 769             {
 770               ip_adjacency_t * iadj0, * iadj1;
 771               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 772               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 773               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 774               dst_addr1 = &iadj1->indirect.next_hop.ip6;
 775             }
 776           else
 777             {
 778               dst_addr0 = &ip0->dst_address;
 779               dst_addr1 = &ip1->dst_address;
 780             }
 781
 782           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 783           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 784
 785           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 786             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 787           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 788             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 789
 790           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 791           adj_index1 = ip6_fib_lookup_with_table (im, fib_index1, dst_addr1);
 792
 793           adj0 = ip_get_adjacency (lm, adj_index0);
 794           adj1 = ip_get_adjacency (lm, adj_index1);
 795
 796           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 797             {
 798               adj_index0 = ip6_fib_lookup_with_table
 799                 (im, adj0->explicit_fib_index, dst_addr0);
 800               adj0 = ip_get_adjacency (lm, adj_index0);
 801             }
 802           if (PREDICT_FALSE (adj1->explicit_fib_index != ~0))
 803             {
 804               adj_index1 = ip6_fib_lookup_with_table
 805                 (im, adj1->explicit_fib_index, dst_addr1);
 806               adj1 = ip_get_adjacency (lm, adj_index1);
 807             }
 808
 809           next0 = adj0->lookup_next_index;
 810           next1 = adj1->lookup_next_index;
 811
 812           /* Only process the HBH Option Header if explicitly configured to do so */
 813           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 814             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 815           next1 = (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 816             adj_index1 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj1->lookup_next_index;
 817
 818           vnet_buffer (p0)->ip.flow_hash =
 819             vnet_buffer(p1)->ip.flow_hash = 0;
 820
 821           if (PREDICT_FALSE(adj0->n_adj > 1))
 822             {
 823               flow_hash_config0 =
 824                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 825               vnet_buffer (p0)->ip.flow_hash =
 826                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 827             }
 828
 829           if (PREDICT_FALSE(adj1->n_adj > 1))
 830             {
 831               flow_hash_config1 =
 832                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 833
 834               vnet_buffer (p1)->ip.flow_hash =
 835                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 836             }
 837
 838           ASSERT (adj0->n_adj > 0);
 839           ASSERT (adj1->n_adj > 0);
 840           ASSERT (is_pow2 (adj0->n_adj));
 841           ASSERT (is_pow2 (adj1->n_adj));
 842           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 843           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
 844
 845           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 846           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 847
 848           vlib_increment_combined_counter
 849               (cm, cpu_index, adj_index0, 1,
 850                vlib_buffer_length_in_chain (vm, p0));
 851           vlib_increment_combined_counter
 852               (cm, cpu_index, adj_index1, 1,
 853                vlib_buffer_length_in_chain (vm, p1));
 854
 855           from += 2;
 856           to_next += 2;
 857           n_left_to_next -= 2;
 858           n_left_from -= 2;
 859
 860           wrong_next = (next0 != next) + 2*(next1 != next);
 861           if (PREDICT_FALSE (wrong_next != 0))
 862             {
 863               switch (wrong_next)
 864                 {
 865                 case 1:
 866                   /* A B A */
 867                   to_next[-2] = pi1;
 868                   to_next -= 1;
 869                   n_left_to_next += 1;
 870                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 871                   break;
 872
 873                 case 2:
 874                   /* A A B */
 875                   to_next -= 1;
 876                   n_left_to_next += 1;
 877                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 878                   break;
 879
 880                 case 3:
 881                   /* A B C */
 882                   to_next -= 2;
 883                   n_left_to_next += 2;
 884                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 885                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 886                   if (next0 == next1)
 887                     {
 888                       /* A B B */
 889                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 890                       next = next1;
 891                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 892                     }
 893                 }
 894             }
 895         }
 896
 897       while (n_left_from > 0 && n_left_to_next > 0)
 898         {
 899           vlib_buffer_t * p0;
 900           ip6_header_t * ip0;
 901           u32 pi0, adj_index0;
 902           ip_lookup_next_t next0;
 903           ip_adjacency_t * adj0;
 904           ip6_address_t * dst_addr0;
 905           u32 fib_index0, flow_hash_config0;
 906
 907           pi0 = from[0];
 908           to_next[0] = pi0;
 909
 910           p0 = vlib_get_buffer (vm, pi0);
 911
 912           ip0 = vlib_buffer_get_current (p0);
 913
 914           if (is_indirect)
 915             {
 916               ip_adjacency_t * iadj0;
 917               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 918               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 919             }
 920           else
 921             {
 922               dst_addr0 = &ip0->dst_address;
 923             }
 924
 925           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 926           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 927             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 928
 929           flow_hash_config0 =
 930               vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 931
 932           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 933
 934           adj0 = ip_get_adjacency (lm, adj_index0);
 935
 936           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 937             {
 938               adj_index0 = ip6_fib_lookup_with_table
 939                 (im, adj0->explicit_fib_index, dst_addr0);
 940               adj0 = ip_get_adjacency (lm, adj_index0);
 941             }
 942
 943           /* Only process the HBH Option Header if explicitly configured to do so */
 944           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 945             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 946
 947           vnet_buffer (p0)->ip.flow_hash = 0;
 948
 949           if (PREDICT_FALSE(adj0->n_adj > 1))
 950             {
 951               flow_hash_config0 =
 952                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 953               vnet_buffer (p0)->ip.flow_hash =
 954                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 955             }
 956
 957           ASSERT (adj0->n_adj > 0);
 958           ASSERT (is_pow2 (adj0->n_adj));
 959           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 960
 961           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 962
 963           vlib_increment_combined_counter
 964               (cm, cpu_index, adj_index0, 1,
 965                vlib_buffer_length_in_chain (vm, p0));
 966
 967           from += 1;
 968           to_next += 1;
 969           n_left_to_next -= 1;
 970           n_left_from -= 1;
 971
 972           if (PREDICT_FALSE (next0 != next))
 973             {
 974               n_left_to_next += 1;
 975               vlib_put_next_frame (vm, node, next, n_left_to_next);
 976               next = next0;
 977               vlib_get_next_frame (vm, node, next,
 978                                    to_next, n_left_to_next);
 979               to_next[0] = pi0;
 980               to_next += 1;
 981               n_left_to_next -= 1;
 982             }
 983         }
 984
 985       vlib_put_next_frame (vm, node, next, n_left_to_next);
 986     }
 987
 988   if (node->flags & VLIB_NODE_FLAG_TRACE)
 989       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 990
 991   return frame->n_vectors;
 992 }
 993
 994 void ip6_adjacency_set_interface_route (vnet_main_t * vnm,
 995                                         ip_adjacency_t * adj,
 996                                         u32 sw_if_index,
 997                                         u32 if_address_index)
 998 {
 999   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1000   ip_lookup_next_t n;
1001   u32 node_index;
1002
1003   if (hw->hw_class_index == ethernet_hw_interface_class.index
1004       || hw->hw_class_index == srp_hw_interface_class.index)
1005     {
1006       n = IP_LOOKUP_NEXT_ARP;
1007       node_index = ip6_discover_neighbor_node.index;
1008       adj->if_address_index = if_address_index;
1009       adj->arp.next_hop.ip6.as_u64[0] = 0;
1010       adj->arp.next_hop.ip6.as_u64[1] = 0;
1011   }
1012   else
1013     {
1014       n = IP_LOOKUP_NEXT_REWRITE;
1015       node_index = ip6_rewrite_node.index;
1016     }
1017
1018  adj->lookup_next_index = n;
1019  adj->explicit_fib_index = ~0;
1020
1021  vnet_rewrite_for_sw_interface
1022    (vnm,
1023     VNET_L3_PACKET_TYPE_IP6,
1024     sw_if_index,
1025     node_index,
1026     VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1027     &adj->rewrite_header,
1028     sizeof (adj->rewrite_data));
1029 }
1030
1031 static void
1032 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
1033                           ip6_main_t * im, u32 fib_index,
1034                           ip_interface_address_t * a)
1035 {
1036   ip_lookup_main_t * lm = &im->lookup_main;
1037   ip_adjacency_t * adj;
1038   ip6_address_t * address = ip_interface_address_get_address (lm, a);
1039   ip6_add_del_route_args_t x;
1040   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1041   u32 classify_table_index;
1042
1043   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1044   x.table_index_or_table_id = fib_index;
1045   x.flags = (IP6_ROUTE_FLAG_ADD
1046              | IP6_ROUTE_FLAG_FIB_INDEX
1047              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1048   x.dst_address = address[0];
1049   x.dst_address_length = a->address_length;
1050   x.n_add_adj = 0;
1051   x.add_adj = 0;
1052
1053   a->neighbor_probe_adj_index = ~0;
1054   if (a->address_length < 128)
1055     {
1056       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1057                               &x.adj_index);
1058       ip6_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1059       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1060       ip6_add_del_route (im, &x);
1061       a->neighbor_probe_adj_index = x.adj_index;
1062     }
1063
1064   /* Add e.g. ::1/128 as local to this host. */
1065   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1066                           &x.adj_index);
1067
1068   classify_table_index = ~0;
1069   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1070     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1071   if (classify_table_index != (u32) ~0)
1072     {
1073       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1074       adj->classify.table_index = classify_table_index;
1075     }
1076   else
1077     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1078
1079   adj->if_address_index = a - lm->if_address_pool;
1080   adj->rewrite_header.sw_if_index = sw_if_index;
1081   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1082   adj->rewrite_header.data_bytes = 0;
1083   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1084   x.dst_address_length = 128;
1085   ip6_add_del_route (im, &x);
1086 }
1087
1088 static void
1089 ip6_del_interface_routes (ip6_main_t * im, u32 fib_index,
1090                           ip6_address_t * address, u32 address_length)
1091 {
1092   ip6_add_del_route_args_t x;
1093
1094   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1095   x.table_index_or_table_id = fib_index;
1096   x.flags = (IP6_ROUTE_FLAG_DEL
1097              | IP6_ROUTE_FLAG_FIB_INDEX
1098              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1099   x.dst_address = address[0];
1100   x.dst_address_length = address_length;
1101   x.adj_index = ~0;
1102   x.n_add_adj = 0;
1103   x.add_adj = 0;
1104
1105   if (address_length < 128)
1106     {
1107       /* Don't wipe out fe80::0/64 */
1108       if (address_length != 64 ||
1109           address[0].as_u64[0] != clib_net_to_host_u64(0xfe80000000000000ULL))
1110         ip6_add_del_route (im, &x);
1111     }
1112
1113   x.dst_address_length = 128;
1114   ip6_add_del_route (im, &x);
1115
1116   ip6_delete_matching_routes (im,
1117                               fib_index,
1118                               IP6_ROUTE_FLAG_FIB_INDEX,
1119                               address,
1120                               address_length);
1121 }
1122
1123 typedef struct {
1124     u32 sw_if_index;
1125     ip6_address_t address;
1126     u32 length;
1127 } ip6_interface_address_t;
1128
1129 static clib_error_t *
1130 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1131                                         u32 sw_if_index,
1132                                         ip6_address_t * new_address,
1133                                         u32 new_length,
1134                                         u32 redistribute,
1135                                         u32 insert_routes,
1136                                         u32 is_del);
1137
1138 static clib_error_t *
1139 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1140                                         u32 sw_if_index,
1141                                         ip6_address_t * address,
1142                                         u32 address_length,
1143                                         u32 redistribute,
1144                                         u32 insert_routes,
1145                                         u32 is_del)
1146 {
1147   vnet_main_t * vnm = vnet_get_main();
1148   ip6_main_t * im = &ip6_main;
1149   ip_lookup_main_t * lm = &im->lookup_main;
1150   clib_error_t * error;
1151   u32 if_address_index;
1152   ip6_address_fib_t ip6_af, * addr_fib = 0;
1153
1154   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1155   ip6_addr_fib_init (&ip6_af, address,
1156                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1157   vec_add1 (addr_fib, ip6_af);
1158
1159   {
1160     uword elts_before = pool_elts (lm->if_address_pool);
1161
1162     error = ip_interface_address_add_del
1163       (lm,
1164        sw_if_index,
1165        addr_fib,
1166        address_length,
1167        is_del,
1168        &if_address_index);
1169     if (error)
1170       goto done;
1171
1172     /* Pool did not grow: add duplicate address. */
1173     if (elts_before == pool_elts (lm->if_address_pool))
1174       goto done;
1175   }
1176
1177   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1178     {
1179       if (is_del)
1180         ip6_del_interface_routes (im, ip6_af.fib_index, address,
1181                                   address_length);
1182
1183       else
1184         ip6_add_interface_routes (vnm, sw_if_index,
1185                                   im, ip6_af.fib_index,
1186                                   pool_elt_at_index (lm->if_address_pool, if_address_index));
1187     }
1188
1189   {
1190     ip6_add_del_interface_address_callback_t * cb;
1191     vec_foreach (cb, im->add_del_interface_address_callbacks)
1192       cb->function (im, cb->function_opaque, sw_if_index,
1193                     address, address_length,
1194                     if_address_index,
1195                     is_del);
1196   }
1197
1198  done:
1199   vec_free (addr_fib);
1200   return error;
1201 }
1202
1203 clib_error_t *
1204 ip6_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1205                                ip6_address_t * address, u32 address_length,
1206                                u32 is_del)
1207 {
1208   return ip6_add_del_interface_address_internal
1209     (vm, sw_if_index, address, address_length,
1210      /* redistribute */ 1,
1211      /* insert_routes */ 1,
1212      is_del);
1213 }
1214
1215 clib_error_t *
1216 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
1217                                 u32 sw_if_index,
1218                                 u32 flags)
1219 {
1220   ip6_main_t * im = &ip6_main;
1221   ip_interface_address_t * ia;
1222   ip6_address_t * a;
1223   u32 is_admin_up, fib_index;
1224
1225   /* Fill in lookup tables with default table (0). */
1226   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1227
1228   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1229
1230   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1231
1232   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1233
1234   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1235                                 0 /* honor unnumbered */,
1236   ({
1237     a = ip_interface_address_get_address (&im->lookup_main, ia);
1238     if (is_admin_up)
1239       ip6_add_interface_routes (vnm, sw_if_index,
1240                                 im, fib_index,
1241                                 ia);
1242     else
1243       ip6_del_interface_routes (im, fib_index,
1244                                 a, ia->address_length);
1245   }));
1246
1247   return 0;
1248 }
1249
1250 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
1251
1252 /* Built-in ip6 unicast rx feature path definition */
1253 VNET_IP6_UNICAST_FEATURE_INIT (ip6_inacl, static) = {
1254   .node_name = "ip6-inacl",
1255   .runs_before = {"ip6-policer-classify", 0},
1256   .feature_index = &ip6_main.ip6_unicast_rx_feature_check_access,
1257 };
1258
1259 VNET_IP6_UNICAST_FEATURE_INIT (ip6_policer_classify, static) = {
1260   .node_name = "ip6-policer-classify",
1261   .runs_before = {"ipsec-input-ip6", 0},
1262   .feature_index = &ip6_main.ip6_unicast_rx_feature_policer_classify,
1263 };
1264
1265 VNET_IP6_UNICAST_FEATURE_INIT (ip6_ipsec, static) = {
1266   .node_name = "ipsec-input-ip6",
1267   .runs_before = {"l2tp-decap", 0},
1268   .feature_index = &ip6_main.ip6_unicast_rx_feature_ipsec,
1269 };
1270
1271 VNET_IP6_UNICAST_FEATURE_INIT (ip6_l2tp, static) = {
1272   .node_name = "l2tp-decap",
1273   .runs_before = {"vpath-input-ip6", 0},
1274   .feature_index = &ip6_main.ip6_unicast_rx_feature_l2tp_decap,
1275 };
1276
1277 VNET_IP6_UNICAST_FEATURE_INIT (ip6_vpath, static) = {
1278   .node_name = "vpath-input-ip6",
1279   .runs_before = {"ip6-lookup", 0},
1280   .feature_index = &ip6_main.ip6_unicast_rx_feature_vpath,
1281 };
1282
1283 VNET_IP6_UNICAST_FEATURE_INIT (ip6_lookup, static) = {
1284   .node_name = "ip6-lookup",
1285   .runs_before = {0}, /* not before any other features */
1286   .feature_index = &ip6_main.ip6_unicast_rx_feature_lookup,
1287 };
1288
1289 /* Built-in ip6 multicast rx feature path definition (none now) */
1290 VNET_IP6_MULTICAST_FEATURE_INIT (ip4_vpath_mc, static) = {
1291   .node_name = "vpath-input-ip6",
1292   .runs_before = {"ip6-lookup", 0},
1293   .feature_index = &ip6_main.ip6_multicast_rx_feature_vpath,
1294 };
1295
1296 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_lookup, static) = {
1297   .node_name = "ip6-lookup",
1298   .runs_before = {0}, /* not before any other features */
1299   .feature_index = &ip6_main.ip6_multicast_rx_feature_lookup,
1300 };
1301
1302 static char * feature_start_nodes[] =
1303   {"ip6-input"};
1304
1305 static clib_error_t *
1306 ip6_feature_init (vlib_main_t * vm, ip6_main_t * im)
1307 {
1308   ip_lookup_main_t * lm = &im->lookup_main;
1309   clib_error_t * error;
1310   vnet_cast_t cast;
1311
1312   for (cast = 0; cast < VNET_N_CAST; cast++)
1313     {
1314       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1315       vnet_config_main_t * vcm = &cm->config_main;
1316
1317       if ((error = ip_feature_init_cast (vm, cm, vcm,
1318                                          feature_start_nodes,
1319                                          ARRAY_LEN(feature_start_nodes),
1320                                          cast,
1321                                          0 /* is_ip4 */)))
1322         return error;
1323     }
1324   return 0;
1325 }
1326
1327 clib_error_t *
1328 ip6_sw_interface_add_del (vnet_main_t * vnm,
1329                           u32 sw_if_index,
1330                           u32 is_add)
1331 {
1332   vlib_main_t * vm = vnm->vlib_main;
1333   ip6_main_t * im = &ip6_main;
1334   ip_lookup_main_t * lm = &im->lookup_main;
1335   u32 ci, cast;
1336   u32 feature_index;
1337
1338   for (cast = 0; cast < VNET_N_CAST; cast++)
1339     {
1340       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1341       vnet_config_main_t * vcm = &cm->config_main;
1342
1343       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1344       ci = cm->config_index_by_sw_if_index[sw_if_index];
1345
1346       if (cast == VNET_UNICAST)
1347         feature_index = im->ip6_unicast_rx_feature_lookup;
1348       else
1349         feature_index = im->ip6_multicast_rx_feature_lookup;
1350
1351       if (is_add)
1352         ci = vnet_config_add_feature (vm, vcm,
1353                                       ci,
1354                                       feature_index,
1355                                       /* config data */ 0,
1356                                       /* # bytes of config data */ 0);
1357       else
1358         ci = vnet_config_del_feature (vm, vcm,
1359                                       ci,
1360                                       feature_index,
1361                                       /* config data */ 0,
1362                                       /* # bytes of config data */ 0);
1363
1364       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1365     }
1366   return /* no error */ 0;
1367 }
1368
1369 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
1370
1371 static uword
1372 ip6_lookup (vlib_main_t * vm,
1373             vlib_node_runtime_t * node,
1374             vlib_frame_t * frame)
1375 {
1376   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 0);
1377 }
1378
1379 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
1380
1381 VLIB_REGISTER_NODE (ip6_lookup_node) = {
1382   .function = ip6_lookup,
1383   .name = "ip6-lookup",
1384   .vector_size = sizeof (u32),
1385
1386   .format_trace = format_ip6_lookup_trace,
1387
1388   .n_next_nodes = IP6_LOOKUP_N_NEXT,
1389   .next_nodes = IP6_LOOKUP_NEXT_NODES,
1390 };
1391
1392 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
1393
1394 static uword
1395 ip6_indirect (vlib_main_t * vm,
1396               vlib_node_runtime_t * node,
1397               vlib_frame_t * frame)
1398 {
1399   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 1);
1400 }
1401
1402
1403 VLIB_REGISTER_NODE (ip6_indirect_node) = {
1404   .function = ip6_indirect,
1405   .name = "ip6-indirect",
1406   .vector_size = sizeof (u32),
1407   .sibling_of = "ip6-lookup",
1408   .format_trace = format_ip6_lookup_trace,
1409   .n_next_nodes = 0,
1410 };
1411
1412 VLIB_NODE_FUNCTION_MULTIARCH (ip6_indirect_node, ip6_indirect)
1413
1414 typedef struct {
1415   /* Adjacency taken. */
1416   u32 adj_index;
1417   u32 flow_hash;
1418   u32 fib_index;
1419
1420   /* Packet data, possibly *after* rewrite. */
1421   u8 packet_data[128 - 1*sizeof(u32)];
1422 } ip6_forward_next_trace_t;
1423
1424 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
1425 {
1426   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1427   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1428   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1429   uword indent = format_get_indent (s);
1430
1431   s = format(s, "%U%U",
1432              format_white_space, indent,
1433              format_ip6_header, t->packet_data);
1434   return s;
1435 }
1436
1437 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
1438 {
1439   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1440   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1441   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1442   vnet_main_t * vnm = vnet_get_main();
1443   ip6_main_t * im = &ip6_main;
1444   uword indent = format_get_indent (s);
1445
1446   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1447               t->fib_index, t->adj_index, format_ip_adjacency,
1448               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1449   s = format(s, "\n%U%U",
1450              format_white_space, indent,
1451              format_ip6_header, t->packet_data);
1452   return s;
1453 }
1454
1455
1456 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
1457 {
1458   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1459   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1460   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1461   vnet_main_t * vnm = vnet_get_main();
1462   ip6_main_t * im = &ip6_main;
1463   uword indent = format_get_indent (s);
1464
1465   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1466               t->fib_index, t->adj_index, format_ip_adjacency,
1467               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1468   s = format (s, "\n%U%U",
1469               format_white_space, indent,
1470               format_ip_adjacency_packet_data,
1471               vnm, &im->lookup_main, t->adj_index,
1472               t->packet_data, sizeof (t->packet_data));
1473   return s;
1474 }
1475
1476 /* Common trace function for all ip6-forward next nodes. */
1477 void
1478 ip6_forward_next_trace (vlib_main_t * vm,
1479                         vlib_node_runtime_t * node,
1480                         vlib_frame_t * frame,
1481                         vlib_rx_or_tx_t which_adj_index)
1482 {
1483   u32 * from, n_left;
1484   ip6_main_t * im = &ip6_main;
1485
1486   n_left = frame->n_vectors;
1487   from = vlib_frame_vector_args (frame);
1488
1489   while (n_left >= 4)
1490     {
1491       u32 bi0, bi1;
1492       vlib_buffer_t * b0, * b1;
1493       ip6_forward_next_trace_t * t0, * t1;
1494
1495       /* Prefetch next iteration. */
1496       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1497       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1498
1499       bi0 = from[0];
1500       bi1 = from[1];
1501
1502       b0 = vlib_get_buffer (vm, bi0);
1503       b1 = vlib_get_buffer (vm, bi1);
1504
1505       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1506         {
1507           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1508           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1509           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1510           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1511               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1512               vec_elt (im->fib_index_by_sw_if_index,
1513                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1514
1515           clib_memcpy (t0->packet_data,
1516                   vlib_buffer_get_current (b0),
1517                   sizeof (t0->packet_data));
1518         }
1519       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1520         {
1521           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1522           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1523           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1524           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1525               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1526               vec_elt (im->fib_index_by_sw_if_index,
1527                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1528
1529           clib_memcpy (t1->packet_data,
1530                   vlib_buffer_get_current (b1),
1531                   sizeof (t1->packet_data));
1532         }
1533       from += 2;
1534       n_left -= 2;
1535     }
1536
1537   while (n_left >= 1)
1538     {
1539       u32 bi0;
1540       vlib_buffer_t * b0;
1541       ip6_forward_next_trace_t * t0;
1542
1543       bi0 = from[0];
1544
1545       b0 = vlib_get_buffer (vm, bi0);
1546
1547       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1548         {
1549           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1550           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1551           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1552           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1553               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1554               vec_elt (im->fib_index_by_sw_if_index,
1555                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1556
1557           clib_memcpy (t0->packet_data,
1558                   vlib_buffer_get_current (b0),
1559                   sizeof (t0->packet_data));
1560         }
1561       from += 1;
1562       n_left -= 1;
1563     }
1564 }
1565
1566 static uword
1567 ip6_drop_or_punt (vlib_main_t * vm,
1568                   vlib_node_runtime_t * node,
1569                   vlib_frame_t * frame,
1570                   ip6_error_t error_code)
1571 {
1572   u32 * buffers = vlib_frame_vector_args (frame);
1573   uword n_packets = frame->n_vectors;
1574
1575   vlib_error_drop_buffers (vm, node,
1576                            buffers,
1577                            /* stride */ 1,
1578                            n_packets,
1579                            /* next */ 0,
1580                            ip6_input_node.index,
1581                            error_code);
1582
1583   if (node->flags & VLIB_NODE_FLAG_TRACE)
1584     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1585
1586   return n_packets;
1587 }
1588
1589 static uword
1590 ip6_drop (vlib_main_t * vm,
1591           vlib_node_runtime_t * node,
1592           vlib_frame_t * frame)
1593 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1594
1595 static uword
1596 ip6_punt (vlib_main_t * vm,
1597           vlib_node_runtime_t * node,
1598           vlib_frame_t * frame)
1599 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1600
1601 static uword
1602 ip6_miss (vlib_main_t * vm,
1603           vlib_node_runtime_t * node,
1604           vlib_frame_t * frame)
1605 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_DST_LOOKUP_MISS); }
1606
1607 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1608   .function = ip6_drop,
1609   .name = "ip6-drop",
1610   .vector_size = sizeof (u32),
1611
1612   .format_trace = format_ip6_forward_next_trace,
1613
1614   .n_next_nodes = 1,
1615   .next_nodes = {
1616     [0] = "error-drop",
1617   },
1618 };
1619
1620 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1621
1622 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1623   .function = ip6_punt,
1624   .name = "ip6-punt",
1625   .vector_size = sizeof (u32),
1626
1627   .format_trace = format_ip6_forward_next_trace,
1628
1629   .n_next_nodes = 1,
1630   .next_nodes = {
1631     [0] = "error-punt",
1632   },
1633 };
1634
1635 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1636
1637 VLIB_REGISTER_NODE (ip6_miss_node,static) = {
1638   .function = ip6_miss,
1639   .name = "ip6-miss",
1640   .vector_size = sizeof (u32),
1641
1642   .format_trace = format_ip6_forward_next_trace,
1643
1644   .n_next_nodes = 1,
1645   .next_nodes = {
1646     [0] = "error-drop",
1647   },
1648 };
1649
1650 VLIB_NODE_FUNCTION_MULTIARCH (ip6_miss_node, ip6_miss)
1651
1652 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1653   .function = ip6_drop,
1654   .name = "ip6-multicast",
1655   .vector_size = sizeof (u32),
1656
1657   .format_trace = format_ip6_forward_next_trace,
1658
1659   .n_next_nodes = 1,
1660   .next_nodes = {
1661     [0] = "error-drop",
1662   },
1663 };
1664
1665 /* Compute TCP/UDP/ICMP6 checksum in software. */
1666 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1667 {
1668   ip_csum_t sum0;
1669   u16 sum16, payload_length_host_byte_order;
1670   u32 i, n_this_buffer, n_bytes_left;
1671   u32 headers_size = sizeof(ip0[0]);
1672   void * data_this_buffer;
1673
1674   ASSERT(bogus_lengthp);
1675   *bogus_lengthp = 0;
1676
1677   /* Initialize checksum with ip header. */
1678   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1679   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1680   data_this_buffer = (void *) (ip0 + 1);
1681
1682   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1683     {
1684       sum0 = ip_csum_with_carry (sum0,
1685                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1686       sum0 = ip_csum_with_carry (sum0,
1687                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1688     }
1689
1690   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1691   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1692     {
1693       u32  skip_bytes;
1694       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1695
1696       /* validate really icmp6 next */
1697       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1698
1699       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1700       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1701
1702       payload_length_host_byte_order  -= skip_bytes;
1703       headers_size += skip_bytes;
1704    }
1705
1706   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1707 #if DPDK > 0
1708   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1709   {
1710     struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
1711     u8 nb_segs = mb->nb_segs;
1712
1713     n_this_buffer = (p0->current_length > headers_size ?
1714                      p0->current_length - headers_size : 0);
1715     while (n_bytes_left)
1716       {
1717         sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1718         n_bytes_left -= n_this_buffer;
1719
1720         mb = mb->next;
1721         nb_segs--;
1722         if ((nb_segs == 0) || (mb == 0))
1723           break;
1724
1725         data_this_buffer = rte_ctrlmbuf_data(mb);
1726         n_this_buffer = mb->data_len;
1727       }
1728     if (n_bytes_left || nb_segs)
1729       {
1730         *bogus_lengthp = 1;
1731         return 0xfefe;
1732       }
1733   }
1734   else sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1735 #else
1736   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1737     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1738   while (1)
1739     {
1740       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1741       n_bytes_left -= n_this_buffer;
1742       if (n_bytes_left == 0)
1743         break;
1744
1745       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1746         {
1747           *bogus_lengthp = 1;
1748           return 0xfefe;
1749         }
1750       p0 = vlib_get_buffer (vm, p0->next_buffer);
1751       data_this_buffer = vlib_buffer_get_current (p0);
1752       n_this_buffer = p0->current_length;
1753     }
1754 #endif /* DPDK */
1755
1756   sum16 = ~ ip_csum_fold (sum0);
1757
1758   return sum16;
1759 }
1760
1761 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1762 {
1763   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1764   udp_header_t * udp0;
1765   u16 sum16;
1766   int bogus_length;
1767
1768   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1769   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1770           || ip0->protocol == IP_PROTOCOL_ICMP6
1771           || ip0->protocol == IP_PROTOCOL_UDP
1772           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1773
1774   udp0 = (void *) (ip0 + 1);
1775   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1776     {
1777       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1778                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1779       return p0->flags;
1780     }
1781
1782   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1783
1784   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1785                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1786
1787   return p0->flags;
1788 }
1789
1790 static uword
1791 ip6_local (vlib_main_t * vm,
1792            vlib_node_runtime_t * node,
1793            vlib_frame_t * frame)
1794 {
1795   ip6_main_t * im = &ip6_main;
1796   ip_lookup_main_t * lm = &im->lookup_main;
1797   ip_local_next_t next_index;
1798   u32 * from, * to_next, n_left_from, n_left_to_next;
1799   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1800
1801   from = vlib_frame_vector_args (frame);
1802   n_left_from = frame->n_vectors;
1803   next_index = node->cached_next_index;
1804
1805   if (node->flags & VLIB_NODE_FLAG_TRACE)
1806     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1807
1808   while (n_left_from > 0)
1809     {
1810       vlib_get_next_frame (vm, node, next_index,
1811                            to_next, n_left_to_next);
1812
1813       while (n_left_from >= 4 && n_left_to_next >= 2)
1814         {
1815           vlib_buffer_t * p0, * p1;
1816           ip6_header_t * ip0, * ip1;
1817           udp_header_t * udp0, * udp1;
1818           u32 pi0, ip_len0, udp_len0, flags0, next0;
1819           u32 pi1, ip_len1, udp_len1, flags1, next1;
1820           i32 len_diff0, len_diff1;
1821           u8 error0, type0, good_l4_checksum0;
1822           u8 error1, type1, good_l4_checksum1;
1823
1824           pi0 = to_next[0] = from[0];
1825           pi1 = to_next[1] = from[1];
1826           from += 2;
1827           n_left_from -= 2;
1828           to_next += 2;
1829           n_left_to_next -= 2;
1830
1831           p0 = vlib_get_buffer (vm, pi0);
1832           p1 = vlib_get_buffer (vm, pi1);
1833
1834           ip0 = vlib_buffer_get_current (p0);
1835           ip1 = vlib_buffer_get_current (p1);
1836
1837           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1838           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1839
1840           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1841           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1842
1843           flags0 = p0->flags;
1844           flags1 = p1->flags;
1845
1846           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1847           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1848
1849           udp0 = ip6_next_header (ip0);
1850           udp1 = ip6_next_header (ip1);
1851
1852           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1853           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1854           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1855
1856           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1857           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1858
1859           /* Verify UDP length. */
1860           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1861           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1862           udp_len0 = clib_net_to_host_u16 (udp0->length);
1863           udp_len1 = clib_net_to_host_u16 (udp1->length);
1864
1865           len_diff0 = ip_len0 - udp_len0;
1866           len_diff1 = ip_len1 - udp_len1;
1867
1868           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1869           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1870
1871           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1872                              && ! good_l4_checksum0
1873                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1874             {
1875               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1876               good_l4_checksum0 =
1877                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1878             }
1879           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1880                              && ! good_l4_checksum1
1881                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1882             {
1883               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1884               good_l4_checksum1 =
1885                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1886             }
1887
1888           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1889
1890           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1891           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1892
1893           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1894           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1895           error0 = (! good_l4_checksum0
1896                     ? IP6_ERROR_UDP_CHECKSUM + type0
1897                     : error0);
1898           error1 = (! good_l4_checksum1
1899                     ? IP6_ERROR_UDP_CHECKSUM + type1
1900                     : error1);
1901
1902           /* Drop packets from unroutable hosts. */
1903           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1904           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1905             {
1906               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1907               error0 = (lm->miss_adj_index == src_adj_index0
1908                         ? IP6_ERROR_SRC_LOOKUP_MISS
1909                         : error0);
1910             }
1911           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL && type1 != IP_BUILTIN_PROTOCOL_ICMP)
1912             {
1913               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1914               error1 = (lm->miss_adj_index == src_adj_index1
1915                         ? IP6_ERROR_SRC_LOOKUP_MISS
1916                         : error1);
1917             }
1918
1919           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1920           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1921
1922           p0->error = error_node->errors[error0];
1923           p1->error = error_node->errors[error1];
1924
1925           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1926                                            to_next, n_left_to_next,
1927                                            pi0, pi1, next0, next1);
1928         }
1929
1930       while (n_left_from > 0 && n_left_to_next > 0)
1931         {
1932           vlib_buffer_t * p0;
1933           ip6_header_t * ip0;
1934           udp_header_t * udp0;
1935           u32 pi0, ip_len0, udp_len0, flags0, next0;
1936           i32 len_diff0;
1937           u8 error0, type0, good_l4_checksum0;
1938
1939           pi0 = to_next[0] = from[0];
1940           from += 1;
1941           n_left_from -= 1;
1942           to_next += 1;
1943           n_left_to_next -= 1;
1944
1945           p0 = vlib_get_buffer (vm, pi0);
1946
1947           ip0 = vlib_buffer_get_current (p0);
1948
1949           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1950           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1951
1952           flags0 = p0->flags;
1953
1954           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1955
1956           udp0 = ip6_next_header (ip0);
1957
1958           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1959           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1960
1961           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1962
1963           /* Verify UDP length. */
1964           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1965           udp_len0 = clib_net_to_host_u16 (udp0->length);
1966
1967           len_diff0 = ip_len0 - udp_len0;
1968
1969           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1970
1971           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1972                              && ! good_l4_checksum0
1973                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1974             {
1975               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1976               good_l4_checksum0 =
1977                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1978             }
1979
1980           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1981
1982           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1983
1984           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1985           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1986           error0 = (! good_l4_checksum0
1987                     ? IP6_ERROR_UDP_CHECKSUM + type0
1988                     : error0);
1989
1990           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1991           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1992             {
1993               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1994               error0 = (lm->miss_adj_index == src_adj_index0
1995                         ? IP6_ERROR_SRC_LOOKUP_MISS
1996                         : error0);
1997             }
1998
1999           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
2000
2001           p0->error = error_node->errors[error0];
2002
2003           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2004                                            to_next, n_left_to_next,
2005                                            pi0, next0);
2006         }
2007
2008       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2009     }
2010
2011   return frame->n_vectors;
2012 }
2013
2014 VLIB_REGISTER_NODE (ip6_local_node,static) = {
2015   .function = ip6_local,
2016   .name = "ip6-local",
2017   .vector_size = sizeof (u32),
2018
2019   .format_trace = format_ip6_forward_next_trace,
2020
2021   .n_next_nodes = IP_LOCAL_N_NEXT,
2022   .next_nodes = {
2023     [IP_LOCAL_NEXT_DROP] = "error-drop",
2024     [IP_LOCAL_NEXT_PUNT] = "error-punt",
2025     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
2026     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
2027   },
2028 };
2029
2030 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
2031
2032 void ip6_register_protocol (u32 protocol, u32 node_index)
2033 {
2034   vlib_main_t * vm = vlib_get_main();
2035   ip6_main_t * im = &ip6_main;
2036   ip_lookup_main_t * lm = &im->lookup_main;
2037
2038   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2039   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
2040 }
2041
2042 typedef enum {
2043   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2044   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
2045   IP6_DISCOVER_NEIGHBOR_N_NEXT,
2046 } ip6_discover_neighbor_next_t;
2047
2048 typedef enum {
2049   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
2050   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
2051   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
2052 } ip6_discover_neighbor_error_t;
2053
2054 static uword
2055 ip6_discover_neighbor (vlib_main_t * vm,
2056                        vlib_node_runtime_t * node,
2057                        vlib_frame_t * frame)
2058 {
2059   vnet_main_t * vnm = vnet_get_main();
2060   ip6_main_t * im = &ip6_main;
2061   ip_lookup_main_t * lm = &im->lookup_main;
2062   u32 * from, * to_next_drop;
2063   uword n_left_from, n_left_to_next_drop;
2064   static f64 time_last_seed_change = -1e100;
2065   static u32 hash_seeds[3];
2066   static uword hash_bitmap[256 / BITS (uword)];
2067   f64 time_now;
2068   int bogus_length;
2069
2070   if (node->flags & VLIB_NODE_FLAG_TRACE)
2071     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2072
2073   time_now = vlib_time_now (vm);
2074   if (time_now - time_last_seed_change > 1e-3)
2075     {
2076       uword i;
2077       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2078                                              sizeof (hash_seeds));
2079       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2080         hash_seeds[i] = r[i];
2081
2082       /* Mark all hash keys as been not-seen before. */
2083       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2084         hash_bitmap[i] = 0;
2085
2086       time_last_seed_change = time_now;
2087     }
2088
2089   from = vlib_frame_vector_args (frame);
2090   n_left_from = frame->n_vectors;
2091
2092   while (n_left_from > 0)
2093     {
2094       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2095                            to_next_drop, n_left_to_next_drop);
2096
2097       while (n_left_from > 0 && n_left_to_next_drop > 0)
2098         {
2099           vlib_buffer_t * p0;
2100           ip6_header_t * ip0;
2101           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2102           uword bm0;
2103           ip_adjacency_t * adj0;
2104           vnet_hw_interface_t * hw_if0;
2105           u32 next0;
2106
2107           pi0 = from[0];
2108
2109           p0 = vlib_get_buffer (vm, pi0);
2110
2111           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2112
2113           ip0 = vlib_buffer_get_current (p0);
2114
2115           adj0 = ip_get_adjacency (lm, adj_index0);
2116
2117           if (adj0->arp.next_hop.ip6.as_u64[0] ||
2118               adj0->arp.next_hop.ip6.as_u64[1]) {
2119             ip0->dst_address.as_u64[0] = adj0->arp.next_hop.ip6.as_u64[0];
2120             ip0->dst_address.as_u64[1] = adj0->arp.next_hop.ip6.as_u64[1];
2121           }
2122
2123           a0 = hash_seeds[0];
2124           b0 = hash_seeds[1];
2125           c0 = hash_seeds[2];
2126
2127           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2128           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2129
2130           a0 ^= sw_if_index0;
2131           b0 ^= ip0->dst_address.as_u32[0];
2132           c0 ^= ip0->dst_address.as_u32[1];
2133
2134           hash_v3_mix32 (a0, b0, c0);
2135
2136           b0 ^= ip0->dst_address.as_u32[2];
2137           c0 ^= ip0->dst_address.as_u32[3];
2138
2139           hash_v3_finalize32 (a0, b0, c0);
2140
2141           c0 &= BITS (hash_bitmap) - 1;
2142           c0 = c0 / BITS (uword);
2143           m0 = (uword) 1 << (c0 % BITS (uword));
2144
2145           bm0 = hash_bitmap[c0];
2146           drop0 = (bm0 & m0) != 0;
2147
2148           /* Mark it as seen. */
2149           hash_bitmap[c0] = bm0 | m0;
2150
2151           from += 1;
2152           n_left_from -= 1;
2153           to_next_drop[0] = pi0;
2154           to_next_drop += 1;
2155           n_left_to_next_drop -= 1;
2156
2157           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2158
2159           /* If the interface is link-down, drop the pkt */
2160           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
2161             drop0 = 1;
2162
2163           p0->error =
2164             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
2165                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
2166           if (drop0)
2167             continue;
2168
2169           {
2170             u32 bi0 = 0;
2171             icmp6_neighbor_solicitation_header_t * h0;
2172             vlib_buffer_t * b0;
2173
2174             h0 = vlib_packet_template_get_packet
2175               (vm, &im->discover_neighbor_packet_template, &bi0);
2176
2177             /*
2178              * Build ethernet header.
2179              * Choose source address based on destination lookup
2180              * adjacency.
2181              */
2182             if (ip6_src_address_for_packet (im, p0, &h0->ip.src_address,
2183                                                 sw_if_index0)) {
2184                 //There is no address on the interface
2185                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
2186                 vlib_buffer_free(vm, &bi0, 1);
2187                 continue;
2188             }
2189
2190             /*
2191              * Destination address is a solicited node multicast address.
2192              * We need to fill in
2193              * the low 24 bits with low 24 bits of target's address.
2194              */
2195             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
2196             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
2197             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
2198
2199             h0->neighbor.target_address = ip0->dst_address;
2200
2201             clib_memcpy (h0->link_layer_option.ethernet_address,
2202                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
2203
2204             /* $$$$ appears we need this; why is the checksum non-zero? */
2205             h0->neighbor.icmp.checksum = 0;
2206             h0->neighbor.icmp.checksum =
2207               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
2208                                                  &bogus_length);
2209
2210             ASSERT (bogus_length == 0);
2211
2212             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2213             b0 = vlib_get_buffer (vm, bi0);
2214             vnet_buffer (b0)->sw_if_index[VLIB_TX]
2215               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
2216
2217             /* Add rewrite/encap string. */
2218             vnet_rewrite_one_header (adj0[0], h0,
2219                                      sizeof (ethernet_header_t));
2220             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2221
2222             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
2223
2224             vlib_set_next_frame_buffer (vm, node, next0, bi0);
2225           }
2226         }
2227
2228       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2229                            n_left_to_next_drop);
2230     }
2231
2232   return frame->n_vectors;
2233 }
2234
2235 static char * ip6_discover_neighbor_error_strings[] = {
2236   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
2237   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
2238   = "neighbor solicitations sent",
2239   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
2240     = "no source address for ND solicitation",
2241 };
2242
2243 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
2244   .function = ip6_discover_neighbor,
2245   .name = "ip6-discover-neighbor",
2246   .vector_size = sizeof (u32),
2247
2248   .format_trace = format_ip6_forward_next_trace,
2249
2250   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
2251   .error_strings = ip6_discover_neighbor_error_strings,
2252
2253   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
2254   .next_nodes = {
2255     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
2256     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
2257   },
2258 };
2259
2260 clib_error_t *
2261 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
2262 {
2263   vnet_main_t * vnm = vnet_get_main();
2264   ip6_main_t * im = &ip6_main;
2265   icmp6_neighbor_solicitation_header_t * h;
2266   ip6_address_t * src;
2267   ip_interface_address_t * ia;
2268   ip_adjacency_t * adj;
2269   vnet_hw_interface_t * hi;
2270   vnet_sw_interface_t * si;
2271   vlib_buffer_t * b;
2272   u32 bi = 0;
2273   int bogus_length;
2274
2275   si = vnet_get_sw_interface (vnm, sw_if_index);
2276
2277   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2278     {
2279       return clib_error_return (0, "%U: interface %U down",
2280                                 format_ip6_address, dst,
2281                                 format_vnet_sw_if_index_name, vnm,
2282                                 sw_if_index);
2283     }
2284
2285   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2286   if (! src)
2287     {
2288       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2289       return clib_error_return
2290         (0, "no matching interface address for destination %U (interface %U)",
2291          format_ip6_address, dst,
2292          format_vnet_sw_if_index_name, vnm, sw_if_index);
2293     }
2294
2295   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
2296
2297   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2298
2299   /* Destination address is a solicited node multicast address.  We need to fill in
2300      the low 24 bits with low 24 bits of target's address. */
2301   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
2302   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
2303   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
2304
2305   h->ip.src_address = src[0];
2306   h->neighbor.target_address = dst[0];
2307
2308   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
2309
2310   h->neighbor.icmp.checksum =
2311     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
2312   ASSERT(bogus_length == 0);
2313
2314   b = vlib_get_buffer (vm, bi);
2315   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2316
2317   /* Add encapsulation string for software interface (e.g. ethernet header). */
2318   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2319   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2320   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2321
2322   {
2323     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2324     u32 * to_next = vlib_frame_vector_args (f);
2325     to_next[0] = bi;
2326     f->n_vectors = 1;
2327     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2328   }
2329
2330   return /* no error */ 0;
2331 }
2332
2333 typedef enum {
2334   IP6_REWRITE_NEXT_DROP,
2335   IP6_REWRITE_NEXT_ICMP_ERROR,
2336 } ip6_rewrite_next_t;
2337
2338 always_inline uword
2339 ip6_rewrite_inline (vlib_main_t * vm,
2340                     vlib_node_runtime_t * node,
2341                     vlib_frame_t * frame,
2342                     int rewrite_for_locally_received_packets)
2343 {
2344   ip_lookup_main_t * lm = &ip6_main.lookup_main;
2345   u32 * from = vlib_frame_vector_args (frame);
2346   u32 n_left_from, n_left_to_next, * to_next, next_index;
2347   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
2348   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2349
2350   n_left_from = frame->n_vectors;
2351   next_index = node->cached_next_index;
2352   u32 cpu_index = os_get_cpu_number();
2353
2354   while (n_left_from > 0)
2355     {
2356       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2357
2358       while (n_left_from >= 4 && n_left_to_next >= 2)
2359         {
2360           ip_adjacency_t * adj0, * adj1;
2361           vlib_buffer_t * p0, * p1;
2362           ip6_header_t * ip0, * ip1;
2363           u32 pi0, rw_len0, next0, error0, adj_index0;
2364           u32 pi1, rw_len1, next1, error1, adj_index1;
2365
2366           /* Prefetch next iteration. */
2367           {
2368             vlib_buffer_t * p2, * p3;
2369
2370             p2 = vlib_get_buffer (vm, from[2]);
2371             p3 = vlib_get_buffer (vm, from[3]);
2372
2373             vlib_prefetch_buffer_header (p2, LOAD);
2374             vlib_prefetch_buffer_header (p3, LOAD);
2375
2376             CLIB_PREFETCH (p2->pre_data, 32, STORE);
2377             CLIB_PREFETCH (p3->pre_data, 32, STORE);
2378
2379             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2380             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2381           }
2382
2383           pi0 = to_next[0] = from[0];
2384           pi1 = to_next[1] = from[1];
2385
2386           from += 2;
2387           n_left_from -= 2;
2388           to_next += 2;
2389           n_left_to_next -= 2;
2390
2391           p0 = vlib_get_buffer (vm, pi0);
2392           p1 = vlib_get_buffer (vm, pi1);
2393
2394           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2395           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2396
2397           /* We should never rewrite a pkt using the MISS adjacency */
2398           ASSERT(adj_index0 && adj_index1);
2399
2400           ip0 = vlib_buffer_get_current (p0);
2401           ip1 = vlib_buffer_get_current (p1);
2402
2403           error0 = error1 = IP6_ERROR_NONE;
2404           next0 = next1 = IP6_REWRITE_NEXT_DROP;
2405
2406           if (! rewrite_for_locally_received_packets)
2407             {
2408               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
2409
2410               /* Input node should have reject packets with hop limit 0. */
2411               ASSERT (ip0->hop_limit > 0);
2412               ASSERT (ip1->hop_limit > 0);
2413
2414               hop_limit0 -= 1;
2415               hop_limit1 -= 1;
2416
2417               ip0->hop_limit = hop_limit0;
2418               ip1->hop_limit = hop_limit1;
2419
2420               /*
2421                * If the hop count drops below 1 when forwarding, generate
2422                * an ICMP response.
2423                */
2424               if (PREDICT_FALSE(hop_limit0 <= 0))
2425                 {
2426                   error0 = IP6_ERROR_TIME_EXPIRED;
2427                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2428                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2429                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2430                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2431                 }
2432               if (PREDICT_FALSE(hop_limit1 <= 0))
2433                 {
2434                   error1 = IP6_ERROR_TIME_EXPIRED;
2435                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
2436                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2437                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
2438                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2439                 }
2440             }
2441
2442           adj0 = ip_get_adjacency (lm, adj_index0);
2443           adj1 = ip_get_adjacency (lm, adj_index1);
2444
2445           if (rewrite_for_locally_received_packets)
2446             {
2447               /*
2448                * If someone sends e.g. an icmp6 w/ src = dst = interface addr,
2449                * we end up here with a local adjacency in hand
2450                */
2451               if (PREDICT_FALSE(adj0->lookup_next_index
2452                                 == IP_LOOKUP_NEXT_LOCAL))
2453                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2454               if (PREDICT_FALSE(adj1->lookup_next_index
2455                                 == IP_LOOKUP_NEXT_LOCAL))
2456                 error1 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2457             }
2458
2459           rw_len0 = adj0[0].rewrite_header.data_bytes;
2460           rw_len1 = adj1[0].rewrite_header.data_bytes;
2461
2462           vlib_increment_combined_counter (&lm->adjacency_counters,
2463                                            cpu_index,
2464                                            adj_index0,
2465                                            /* packet increment */ 0,
2466                                            /* byte increment */ rw_len0);
2467           vlib_increment_combined_counter (&lm->adjacency_counters,
2468                                            cpu_index,
2469                                            adj_index1,
2470                                            /* packet increment */ 0,
2471                                            /* byte increment */ rw_len1);
2472
2473           /* Check MTU of outgoing interface. */
2474           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2475                     ? IP6_ERROR_MTU_EXCEEDED
2476                     : error0);
2477           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2478                     ? IP6_ERROR_MTU_EXCEEDED
2479                     : error1);
2480
2481           /* Don't adjust the buffer for hop count issue; icmp-error node
2482            * wants to see the IP headerr */
2483           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2484             {
2485               p0->current_data -= rw_len0;
2486               p0->current_length += rw_len0;
2487
2488               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2489                   adj0[0].rewrite_header.sw_if_index;
2490               next0 = adj0[0].rewrite_header.next_index;
2491             }
2492           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
2493             {
2494               p1->current_data -= rw_len1;
2495               p1->current_length += rw_len1;
2496
2497               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2498                   adj1[0].rewrite_header.sw_if_index;
2499               next1 = adj1[0].rewrite_header.next_index;
2500             }
2501
2502           /* Guess we are only writing on simple Ethernet header. */
2503           vnet_rewrite_two_headers (adj0[0], adj1[0],
2504                                     ip0, ip1,
2505                                     sizeof (ethernet_header_t));
2506
2507           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2508                                            to_next, n_left_to_next,
2509                                            pi0, pi1, next0, next1);
2510         }
2511
2512       while (n_left_from > 0 && n_left_to_next > 0)
2513         {
2514           ip_adjacency_t * adj0;
2515           vlib_buffer_t * p0;
2516           ip6_header_t * ip0;
2517           u32 pi0, rw_len0;
2518           u32 adj_index0, next0, error0;
2519
2520           pi0 = to_next[0] = from[0];
2521
2522           p0 = vlib_get_buffer (vm, pi0);
2523
2524           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2525
2526           /* We should never rewrite a pkt using the MISS adjacency */
2527           ASSERT(adj_index0);
2528
2529           adj0 = ip_get_adjacency (lm, adj_index0);
2530
2531           ip0 = vlib_buffer_get_current (p0);
2532
2533           error0 = IP6_ERROR_NONE;
2534           next0 = IP6_REWRITE_NEXT_DROP;
2535
2536           /* Check hop limit */
2537           if (! rewrite_for_locally_received_packets)
2538             {
2539               i32 hop_limit0 = ip0->hop_limit;
2540
2541               ASSERT (ip0->hop_limit > 0);
2542
2543               hop_limit0 -= 1;
2544
2545               ip0->hop_limit = hop_limit0;
2546
2547               if (PREDICT_FALSE(hop_limit0 <= 0))
2548                 {
2549                   /*
2550                    * If the hop count drops below 1 when forwarding, generate
2551                    * an ICMP response.
2552                    */
2553                   error0 = IP6_ERROR_TIME_EXPIRED;
2554                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2555                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2556                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2557                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2558                 }
2559             }
2560
2561           if (rewrite_for_locally_received_packets)
2562             {
2563               if (PREDICT_FALSE(adj0->lookup_next_index
2564                                 == IP_LOOKUP_NEXT_LOCAL))
2565                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2566             }
2567
2568           /* Guess we are only writing on simple Ethernet header. */
2569           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2570
2571           /* Update packet buffer attributes/set output interface. */
2572           rw_len0 = adj0[0].rewrite_header.data_bytes;
2573
2574           vlib_increment_combined_counter (&lm->adjacency_counters,
2575                                            cpu_index,
2576                                            adj_index0,
2577                                            /* packet increment */ 0,
2578                                            /* byte increment */ rw_len0);
2579
2580           /* Check MTU of outgoing interface. */
2581           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2582                     ? IP6_ERROR_MTU_EXCEEDED
2583                     : error0);
2584
2585           /* Don't adjust the buffer for hop count issue; icmp-error node
2586            * wants to see the IP headerr */
2587           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2588             {
2589               p0->current_data -= rw_len0;
2590               p0->current_length += rw_len0;
2591
2592               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2593                   adj0[0].rewrite_header.sw_if_index;
2594               next0 = adj0[0].rewrite_header.next_index;
2595             }
2596
2597           p0->error = error_node->errors[error0];
2598
2599           from += 1;
2600           n_left_from -= 1;
2601           to_next += 1;
2602           n_left_to_next -= 1;
2603
2604           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2605                                            to_next, n_left_to_next,
2606                                            pi0, next0);
2607         }
2608
2609       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2610     }
2611
2612   /* Need to do trace after rewrites to pick up new packet data. */
2613   if (node->flags & VLIB_NODE_FLAG_TRACE)
2614     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2615
2616   return frame->n_vectors;
2617 }
2618
2619 static uword
2620 ip6_rewrite_transit (vlib_main_t * vm,
2621                      vlib_node_runtime_t * node,
2622                      vlib_frame_t * frame)
2623 {
2624   return ip6_rewrite_inline (vm, node, frame,
2625                              /* rewrite_for_locally_received_packets */ 0);
2626 }
2627
2628 static uword
2629 ip6_rewrite_local (vlib_main_t * vm,
2630                    vlib_node_runtime_t * node,
2631                    vlib_frame_t * frame)
2632 {
2633   return ip6_rewrite_inline (vm, node, frame,
2634                              /* rewrite_for_locally_received_packets */ 1);
2635 }
2636
2637 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2638   .function = ip6_rewrite_transit,
2639   .name = "ip6-rewrite",
2640   .vector_size = sizeof (u32),
2641
2642   .format_trace = format_ip6_rewrite_trace,
2643
2644   .n_next_nodes = 2,
2645   .next_nodes = {
2646     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2647     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2648   },
2649 };
2650
2651 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit)
2652
2653 VLIB_REGISTER_NODE (ip6_rewrite_local_node) = {
2654   .function = ip6_rewrite_local,
2655   .name = "ip6-rewrite-local",
2656   .vector_size = sizeof (u32),
2657
2658   .sibling_of = "ip6-rewrite",
2659
2660   .format_trace = format_ip6_rewrite_trace,
2661
2662   .n_next_nodes = 0,
2663 };
2664
2665 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local)
2666
2667 /*
2668  * Hop-by-Hop handling
2669  */
2670
2671 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2672
2673 #define foreach_ip6_hop_by_hop_error \
2674 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2675 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2676 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2677
2678 typedef enum {
2679 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2680   foreach_ip6_hop_by_hop_error
2681 #undef _
2682   IP6_HOP_BY_HOP_N_ERROR,
2683 } ip6_hop_by_hop_error_t;
2684
2685 /*
2686  * Primary h-b-h handler trace support
2687  * We work pretty hard on the problem for obvious reasons
2688  */
2689 typedef struct {
2690   u32 next_index;
2691   u32 trace_len;
2692   u8 option_data[256];
2693 } ip6_hop_by_hop_trace_t;
2694
2695 vlib_node_registration_t ip6_hop_by_hop_node;
2696
2697 static char * ip6_hop_by_hop_error_strings[] = {
2698 #define _(sym,string) string,
2699   foreach_ip6_hop_by_hop_error
2700 #undef _
2701 };
2702
2703 static u8 *
2704 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2705 {
2706   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2707   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2708   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2709   ip6_hop_by_hop_header_t *hbh0;
2710   ip6_hop_by_hop_option_t *opt0, *limit0;
2711   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2712
2713   u8 type0;
2714
2715   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2716
2717   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2718               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2719
2720   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2721   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2722
2723   while (opt0 < limit0) {
2724     type0 = opt0->type;
2725     switch (type0) {
2726     case 0: /* Pad, just stop */
2727       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2728       break;
2729
2730     default:
2731       if (hm->trace[type0]) {
2732         s = (*hm->trace[type0])(s, opt0);
2733       } else {
2734         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2735       }
2736       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2737       break;
2738     }
2739   }
2740   return s;
2741 }
2742
2743 /*
2744  * Process the Hop-by-Hop Options header
2745  */
2746 static uword
2747 ip6_hop_by_hop (vlib_main_t * vm,
2748                 vlib_node_runtime_t * node,
2749                 vlib_frame_t * frame)
2750 {
2751   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2752   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2753   u32 n_left_from, *from, *to_next;
2754   ip_lookup_next_t next_index;
2755   ip6_main_t * im = &ip6_main;
2756   ip_lookup_main_t *lm = &im->lookup_main;
2757
2758   from = vlib_frame_vector_args (frame);
2759   n_left_from = frame->n_vectors;
2760   next_index = node->cached_next_index;
2761
2762   while (n_left_from > 0) {
2763     u32 n_left_to_next;
2764
2765     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2766
2767     while (n_left_from > 0 && n_left_to_next > 0) {
2768       u32 bi0;
2769       vlib_buffer_t * b0;
2770       u32 next0;
2771       ip6_header_t * ip0;
2772       ip6_hop_by_hop_header_t *hbh0;
2773       ip6_hop_by_hop_option_t *opt0, *limit0;
2774       u8 type0;
2775       u8 error0 = 0;
2776
2777       /* Speculatively enqueue b0 to the current next frame */
2778       bi0 = from[0];
2779       to_next[0] = bi0;
2780       from += 1;
2781       to_next += 1;
2782       n_left_from -= 1;
2783       n_left_to_next -= 1;
2784
2785       b0 = vlib_get_buffer (vm, bi0);
2786       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2787       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2788       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2789       next0 = adj0->lookup_next_index;
2790
2791       ip0 = vlib_buffer_get_current (b0);
2792       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2793       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2794       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2795
2796       /*
2797        * Basic validity checks
2798        */
2799       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2800         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2801         next0 = IP_LOOKUP_NEXT_DROP;
2802         goto out0;
2803       }
2804
2805       /* Scan the set of h-b-h options, process ones that we understand */
2806       while (opt0 < limit0) {
2807         type0 = opt0->type;
2808         switch (type0) {
2809         case 0: /* Pad1 */
2810           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2811           continue;
2812         case 1: /* PadN */
2813           break;
2814         default:
2815           if (hm->options[type0]) {
2816             if ((*hm->options[type0])(b0, ip0, opt0) < 0) {
2817               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2818               goto out0;
2819             }
2820           } else {
2821             /* Unrecognized mandatory option, check the two high order bits */
2822             switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS) {
2823             case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2824               break;
2825             case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2826               next0 = IP_LOOKUP_NEXT_DROP;
2827               break;
2828             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2829               next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2830               icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2831                                           ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2832               break;
2833             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2834               if (!ip6_address_is_multicast(&ip0->dst_address)) {
2835                 next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2836                 icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2837                                             ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2838               } else {
2839                 next0 =  IP_LOOKUP_NEXT_DROP;
2840               }
2841               break;
2842             }
2843             error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2844             goto out0;
2845           }
2846         }
2847         opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2848       }
2849
2850     out0:
2851       /* Has the classifier flagged this buffer for special treatment? */
2852       if ((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index == OI_DECAP))
2853         next0 = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2854
2855       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2856         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2857         u32 trace_len = (hbh0->length + 1) << 3;
2858         t->next_index = next0;
2859         /* Capture the h-b-h option verbatim */
2860         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2861         t->trace_len = trace_len;
2862         clib_memcpy(t->option_data, hbh0, trace_len);
2863       }
2864
2865       b0->error = error_node->errors[error0];
2866
2867       /* verify speculative enqueue, maybe switch current next frame */
2868       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2869     }
2870     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2871   }
2872   return frame->n_vectors;
2873 }
2874
2875 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2876   .function = ip6_hop_by_hop,
2877   .name = "ip6-hop-by-hop",
2878   .sibling_of = "ip6-lookup",
2879   .vector_size = sizeof (u32),
2880   .format_trace = format_ip6_hop_by_hop_trace,
2881   .type = VLIB_NODE_TYPE_INTERNAL,
2882   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2883   .error_strings = ip6_hop_by_hop_error_strings,
2884   .n_next_nodes = 0,
2885 };
2886
2887 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop)
2888
2889 static clib_error_t *
2890 ip6_hop_by_hop_init (vlib_main_t * vm)
2891 {
2892   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2893   memset(hm->options, 0, sizeof(hm->options));
2894   memset(hm->trace, 0, sizeof(hm->trace));
2895
2896   return (0);
2897 }
2898
2899 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2900
2901 int
2902 ip6_hbh_register_option (u8 option,
2903                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2904                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2905 {
2906   ip6_main_t * im = &ip6_main;
2907   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2908
2909   ASSERT (option < ARRAY_LEN (hm->options));
2910
2911   /* Already registered */
2912   if (hm->options[option])
2913     return (-1);
2914
2915   hm->options[option] = options;
2916   hm->trace[option] = trace;
2917
2918   /* Set global variable */
2919   im->hbh_enabled = 1;
2920
2921   return (0);
2922 }
2923
2924 int
2925 ip6_hbh_unregister_option (u8 option)
2926 {
2927   ip6_main_t * im = &ip6_main;
2928   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2929
2930   ASSERT (option < ARRAY_LEN (hm->options));
2931
2932   /* Not registered */
2933   if (!hm->options[option])
2934     return (-1);
2935
2936   hm->options[option] = NULL;
2937   hm->trace[option] = NULL;
2938
2939   /* Disable global knob if this was the last option configured */
2940   int i;
2941   bool found = false;
2942   for (i = 0; i < 256; i++) {
2943     if (hm->options[option]) {
2944       found = true;
2945       break;
2946     }
2947   }
2948   if (!found)
2949     im->hbh_enabled = 0;
2950
2951   return (0);
2952 }
2953
2954 /* Global IP6 main. */
2955 ip6_main_t ip6_main;
2956
2957 static clib_error_t *
2958 ip6_lookup_init (vlib_main_t * vm)
2959 {
2960   ip6_main_t * im = &ip6_main;
2961   clib_error_t * error;
2962   uword i;
2963
2964   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2965     {
2966       u32 j, i0, i1;
2967
2968       i0 = i / 32;
2969       i1 = i % 32;
2970
2971       for (j = 0; j < i0; j++)
2972         im->fib_masks[i].as_u32[j] = ~0;
2973
2974       if (i1)
2975         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2976     }
2977
2978   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2979
2980   if (im->lookup_table_nbuckets == 0)
2981     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2982
2983   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2984
2985   if (im->lookup_table_size == 0)
2986     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2987
2988   BV(clib_bihash_init) (&im->ip6_lookup_table, "ip6 lookup table",
2989                         im->lookup_table_nbuckets,
2990                         im->lookup_table_size);
2991
2992   /* Create FIB with index 0 and table id of 0. */
2993   find_ip6_fib_by_table_index_or_id (im, /* table id */ 0, IP6_ROUTE_FLAG_TABLE_ID);
2994
2995   {
2996     pg_node_t * pn;
2997     pn = pg_get_node (ip6_lookup_node.index);
2998     pn->unformat_edit = unformat_pg_ip6_header;
2999   }
3000
3001   /* Unless explicitly configured, don't process HBH options */
3002   im->hbh_enabled = 0;
3003
3004   {
3005     icmp6_neighbor_solicitation_header_t p;
3006
3007     memset (&p, 0, sizeof (p));
3008
3009     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
3010     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
3011                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
3012     p.ip.protocol = IP_PROTOCOL_ICMP6;
3013     p.ip.hop_limit = 255;
3014     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
3015
3016     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
3017
3018     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
3019     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
3020
3021     vlib_packet_template_init (vm,
3022                                &im->discover_neighbor_packet_template,
3023                                &p, sizeof (p),
3024                                /* alloc chunk size */ 8,
3025                                "ip6 neighbor discovery");
3026   }
3027
3028   error = ip6_feature_init (vm, im);
3029
3030   return error;
3031 }
3032
3033 VLIB_INIT_FUNCTION (ip6_lookup_init);
3034
3035 static clib_error_t *
3036 add_del_ip6_interface_table (vlib_main_t * vm,
3037                              unformat_input_t * input,
3038                              vlib_cli_command_t * cmd)
3039 {
3040   vnet_main_t * vnm = vnet_get_main();
3041   clib_error_t * error = 0;
3042   u32 sw_if_index, table_id;
3043
3044   sw_if_index = ~0;
3045
3046   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
3047     {
3048       error = clib_error_return (0, "unknown interface `%U'",
3049                                  format_unformat_error, input);
3050       goto done;
3051     }
3052
3053   if (unformat (input, "%d", &table_id))
3054     ;
3055   else
3056     {
3057       error = clib_error_return (0, "expected table id `%U'",
3058                                  format_unformat_error, input);
3059       goto done;
3060     }
3061
3062   {
3063     ip6_main_t * im = &ip6_main;
3064     ip6_fib_t * fib =
3065       find_ip6_fib_by_table_index_or_id (im, table_id, IP6_ROUTE_FLAG_TABLE_ID);
3066
3067     if (fib)
3068       {
3069         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
3070         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
3071     }
3072   }
3073
3074  done:
3075   return error;
3076 }
3077
3078 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
3079   .path = "set interface ip6 table",
3080   .function = add_del_ip6_interface_table,
3081   .short_help = "set interface ip6 table <intfc> <table-id>"
3082 };
3083
3084 void
3085 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
3086                                                   u8 *mac)
3087 {
3088   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
3089   /* Invert the "u" bit */
3090   ip->as_u8 [8] = mac[0] ^ (1<<1);
3091   ip->as_u8 [9] = mac[1];
3092   ip->as_u8 [10] = mac[2];
3093   ip->as_u8 [11] = 0xFF;
3094   ip->as_u8 [12] = 0xFE;
3095   ip->as_u8 [13] = mac[3];
3096   ip->as_u8 [14] = mac[4];
3097   ip->as_u8 [15] = mac[5];
3098 }
3099
3100 void
3101 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
3102                                                   ip6_address_t *ip)
3103 {
3104   /* Invert the previously inverted "u" bit */
3105   mac[0] = ip->as_u8 [8] ^ (1<<1);
3106   mac[1] = ip->as_u8 [9];
3107   mac[2] = ip->as_u8 [10];
3108   mac[3] = ip->as_u8 [13];
3109   mac[4] = ip->as_u8 [14];
3110   mac[5] = ip->as_u8 [15];
3111 }
3112
3113 static clib_error_t *
3114 test_ip6_link_command_fn (vlib_main_t * vm,
3115                           unformat_input_t * input,
3116                           vlib_cli_command_t * cmd)
3117 {
3118   u8 mac[6];
3119   ip6_address_t _a, *a = &_a;
3120
3121   if (unformat (input, "%U", unformat_ethernet_address, mac))
3122     {
3123       ip6_link_local_address_from_ethernet_mac_address (a, mac);
3124       vlib_cli_output (vm, "Link local address: %U",
3125                        format_ip6_address, a);
3126       ip6_ethernet_mac_address_from_link_local_address (mac, a);
3127       vlib_cli_output (vm, "Original MAC address: %U",
3128                        format_ethernet_address, mac);
3129     }
3130
3131   return 0;
3132 }
3133
3134 VLIB_CLI_COMMAND (test_link_command, static) = {
3135   .path = "test ip6 link",
3136   .function = test_ip6_link_command_fn,
3137   .short_help = "test ip6 link <mac-address>",
3138 };
3139
3140 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
3141 {
3142   ip6_main_t * im6 = &ip6_main;
3143   ip6_fib_t * fib;
3144   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
3145
3146   if (p == 0)
3147     return -1;
3148
3149   fib = vec_elt_at_index (im6->fibs, p[0]);
3150
3151   fib->flow_hash_config = flow_hash_config;
3152   return 1;
3153 }
3154
3155 static clib_error_t *
3156 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
3157                               unformat_input_t * input,
3158                               vlib_cli_command_t * cmd)
3159 {
3160   int matched = 0;
3161   u32 table_id = 0;
3162   u32 flow_hash_config = 0;
3163   int rv;
3164
3165   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3166     if (unformat (input, "table %d", &table_id))
3167       matched = 1;
3168 #define _(a,v) \
3169     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3170     foreach_flow_hash_bit
3171 #undef _
3172     else break;
3173   }
3174
3175   if (matched == 0)
3176     return clib_error_return (0, "unknown input `%U'",
3177                               format_unformat_error, input);
3178
3179   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
3180   switch (rv)
3181     {
3182     case 1:
3183       break;
3184
3185     case -1:
3186       return clib_error_return (0, "no such FIB table %d", table_id);
3187
3188     default:
3189       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3190       break;
3191     }
3192
3193   return 0;
3194 }
3195
3196 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
3197     .path = "set ip6 flow-hash",
3198     .short_help =
3199     "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3200     .function = set_ip6_flow_hash_command_fn,
3201 };
3202
3203 static clib_error_t *
3204 show_ip6_local_command_fn (vlib_main_t * vm,
3205                            unformat_input_t * input,
3206                            vlib_cli_command_t * cmd)
3207 {
3208   ip6_main_t * im = &ip6_main;
3209   ip_lookup_main_t * lm = &im->lookup_main;
3210   int i;
3211
3212   vlib_cli_output (vm, "Protocols handled by ip6_local");
3213   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
3214     {
3215       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3216         vlib_cli_output (vm, "%d", i);
3217     }
3218   return 0;
3219 }
3220
3221
3222
3223 VLIB_CLI_COMMAND (show_ip_local, static) = {
3224   .path = "show ip6 local",
3225   .function = show_ip6_local_command_fn,
3226   .short_help = "Show ip6 local protocol table",
3227 };
3228
3229 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3230                                  u32 table_index)
3231 {
3232   vnet_main_t * vnm = vnet_get_main();
3233   vnet_interface_main_t * im = &vnm->interface_main;
3234   ip6_main_t * ipm = &ip6_main;
3235   ip_lookup_main_t * lm = &ipm->lookup_main;
3236   vnet_classify_main_t * cm = &vnet_classify_main;
3237
3238   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3239     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3240
3241   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3242     return VNET_API_ERROR_NO_SUCH_ENTRY;
3243
3244   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3245   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3246
3247   return 0;
3248 }
3249
3250 static clib_error_t *
3251 set_ip6_classify_command_fn (vlib_main_t * vm,
3252                              unformat_input_t * input,
3253                              vlib_cli_command_t * cmd)
3254 {
3255   u32 table_index = ~0;
3256   int table_index_set = 0;
3257   u32 sw_if_index = ~0;
3258   int rv;
3259
3260   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3261     if (unformat (input, "table-index %d", &table_index))
3262       table_index_set = 1;
3263     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3264                        vnet_get_main(), &sw_if_index))
3265         ;
3266     else
3267         break;
3268   }
3269
3270   if (table_index_set == 0)
3271       return clib_error_return (0, "classify table-index must be specified");
3272
3273   if (sw_if_index == ~0)
3274     return clib_error_return (0, "interface / subif must be specified");
3275
3276   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3277
3278   switch (rv)
3279     {
3280     case 0:
3281       break;
3282
3283     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3284       return clib_error_return (0, "No such interface");
3285
3286     case VNET_API_ERROR_NO_SUCH_ENTRY:
3287       return clib_error_return (0, "No such classifier table");
3288     }
3289   return 0;
3290 }
3291
3292 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3293     .path = "set ip6 classify",
3294     .short_help =
3295     "set ip6 classify intfc <int> table-index <index>",
3296     .function = set_ip6_classify_command_fn,
3297 };
3298
3299 static clib_error_t *
3300 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3301 {
3302   ip6_main_t * im = &ip6_main;
3303   uword heapsize = 0;
3304   u32 tmp;
3305   u32 nbuckets = 0;
3306
3307   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3308     if (unformat (input, "hash-buckets %d", &tmp))
3309       nbuckets = tmp;
3310     else if (unformat (input, "heap-size %dm", &tmp))
3311       heapsize = ((u64)tmp) << 20;
3312     else if (unformat (input, "heap-size %dM", &tmp))
3313       heapsize = ((u64)tmp) << 20;
3314     else if (unformat (input, "heap-size %dg", &tmp))
3315       heapsize = ((u64)tmp) << 30;
3316     else if (unformat (input, "heap-size %dG", &tmp))
3317       heapsize = ((u64)tmp) << 30;
3318     else
3319       return clib_error_return (0, "unknown input '%U'",
3320                                 format_unformat_error, input);
3321   }
3322
3323   im->lookup_table_nbuckets = nbuckets;
3324   im->lookup_table_size = heapsize;
3325
3326   return 0;
3327 }
3328
3329 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3330