vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45
  46 #include <vppinfra/bihash_template.c>
  47
  48 static void compute_prefix_lengths_in_search_order (ip6_main_t * im)
  49 {
  50   int i;
  51   vec_reset_length (im->prefix_lengths_in_search_order);
  52   /* Note: bitmap reversed so this is in fact a longest prefix match */
  53   clib_bitmap_foreach (i, im->non_empty_dst_address_length_bitmap,
  54   ({
  55     int dst_address_length = 128 - i;
  56     vec_add1 (im->prefix_lengths_in_search_order, dst_address_length);
  57   }));
  58 }
  59
  60 u32
  61 ip6_fib_lookup_with_table (ip6_main_t * im, u32 fib_index, ip6_address_t * dst)
  62 {
  63   ip_lookup_main_t * lm = &im->lookup_main;
  64   int i, len;
  65   int rv;
  66   BVT(clib_bihash_kv) kv, value;
  67   u64 fib;
  68
  69   len = vec_len (im->prefix_lengths_in_search_order);
  70
  71   kv.key[0] = dst->as_u64[0];
  72   kv.key[1] = dst->as_u64[1];
  73   fib = ((u64)((fib_index))<<32);
  74
  75   for (i = 0; i < len; i++)
  76     {
  77       int dst_address_length = im->prefix_lengths_in_search_order[i];
  78       ip6_address_t * mask = &im->fib_masks[dst_address_length];
  79
  80       ASSERT(dst_address_length >= 0 && dst_address_length <= 128);
  81       //As lengths are decreasing, masks are increasingly specific.
  82       kv.key[0] &= mask->as_u64[0];
  83       kv.key[1] &= mask->as_u64[1];
  84       kv.key[2] = fib | dst_address_length;
  85
  86       rv = BV(clib_bihash_search_inline_2)(&im->ip6_lookup_table, &kv, &value);
  87       if (rv == 0)
  88         return value.value;
  89     }
  90
  91   return lm->miss_adj_index;
  92 }
  93
  94 u32 ip6_fib_lookup (ip6_main_t * im, u32 sw_if_index, ip6_address_t * dst)
  95 {
  96     u32 fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
  97     return ip6_fib_lookup_with_table (im, fib_index, dst);
  98 }
  99
 100 void
 101 vnet_ip6_fib_init (ip6_main_t * im, u32 fib_index)
 102 {
 103   ip_lookup_main_t * lm = &im->lookup_main;
 104   ip6_add_del_route_args_t a;
 105   ip_adjacency_t * adj;
 106
 107   memset(&a, 0x0, sizeof(ip6_add_del_route_args_t));
 108
 109   a.table_index_or_table_id = fib_index;
 110   a.flags = (IP6_ROUTE_FLAG_ADD
 111              | IP6_ROUTE_FLAG_FIB_INDEX
 112              | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 113              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
 114
 115   /* Add ff02::1:ff00:0/104 via local route for all tables.
 116      This is required for neighbor discovery to work. */
 117   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 118                           &a.adj_index);
 119   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 120   adj->if_address_index = ~0;
 121   adj->rewrite_header.data_bytes = 0;
 122
 123   ip6_set_solicited_node_multicast_address (&a.dst_address, 0);
 124
 125   a.dst_address_length = 104;
 126   ip6_add_del_route (im, &a);
 127
 128   /* Add all-routers multicast address via local route for all tables */
 129   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 130                           &a.adj_index);
 131   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 132   adj->if_address_index = ~0;
 133   adj->rewrite_header.data_bytes = 0;
 134
 135   ip6_set_reserved_multicast_address (&a.dst_address,
 136                                       IP6_MULTICAST_SCOPE_link_local,
 137                                       IP6_MULTICAST_GROUP_ID_all_routers);
 138
 139   a.dst_address_length = 128;
 140   ip6_add_del_route (im, &a);
 141
 142   /* Add all-nodes multicast address via local route for all tables */
 143   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 144                           &a.adj_index);
 145   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 146   adj->if_address_index = ~0;
 147   adj->rewrite_header.data_bytes = 0;
 148
 149   ip6_set_reserved_multicast_address (&a.dst_address,
 150                                       IP6_MULTICAST_SCOPE_link_local,
 151                                       IP6_MULTICAST_GROUP_ID_all_hosts);
 152
 153   a.dst_address_length = 128;
 154   ip6_add_del_route (im, &a);
 155
 156   /* Add all-mldv2  multicast address via local route for all tables */
 157   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 158                           &a.adj_index);
 159   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 160   adj->if_address_index = ~0;
 161   adj->rewrite_header.data_bytes = 0;
 162
 163   ip6_set_reserved_multicast_address (&a.dst_address,
 164                                       IP6_MULTICAST_SCOPE_link_local,
 165                                       IP6_MULTICAST_GROUP_ID_mldv2_routers);
 166
 167   a.dst_address_length = 128;
 168   ip6_add_del_route (im, &a);
 169 }
 170
 171 static ip6_fib_t *
 172 create_fib_with_table_id (ip6_main_t * im, u32 table_id)
 173 {
 174   ip6_fib_t * fib;
 175   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
 176   vec_add2 (im->fibs, fib, 1);
 177   fib->table_id = table_id;
 178   fib->index = fib - im->fibs;
 179   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
 180   vnet_ip6_fib_init (im, fib->index);
 181   return fib;
 182 }
 183
 184 ip6_fib_t *
 185 find_ip6_fib_by_table_index_or_id (ip6_main_t * im, u32 table_index_or_id, u32 flags)
 186 {
 187   uword * p, fib_index;
 188
 189   fib_index = table_index_or_id;
 190   if (! (flags & IP6_ROUTE_FLAG_FIB_INDEX))
 191     {
 192       if (table_index_or_id == ~0) {
 193         table_index_or_id = 0;
 194         while (hash_get (im->fib_index_by_table_id, table_index_or_id)) {
 195           table_index_or_id++;
 196         }
 197         return create_fib_with_table_id (im, table_index_or_id);
 198       }
 199
 200       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 201       if (! p)
 202         return create_fib_with_table_id (im, table_index_or_id);
 203       fib_index = p[0];
 204     }
 205   return vec_elt_at_index (im->fibs, fib_index);
 206 }
 207
 208 void ip6_add_del_route (ip6_main_t * im, ip6_add_del_route_args_t * a)
 209 {
 210   ip_lookup_main_t * lm = &im->lookup_main;
 211   ip6_fib_t * fib;
 212   ip6_address_t dst_address;
 213   u32 dst_address_length, adj_index;
 214   uword is_del;
 215   u32 old_adj_index = ~0;
 216   BVT(clib_bihash_kv) kv, value;
 217
 218   vlib_smp_unsafe_warning();
 219
 220   is_del = (a->flags & IP6_ROUTE_FLAG_DEL) != 0;
 221
 222   /* Either create new adjacency or use given one depending on arguments. */
 223   if (a->n_add_adj > 0)
 224     {
 225       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 226       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 227     }
 228   else
 229     adj_index = a->adj_index;
 230
 231   dst_address = a->dst_address;
 232   dst_address_length = a->dst_address_length;
 233   fib = find_ip6_fib_by_table_index_or_id (im, a->table_index_or_table_id,
 234                                            a->flags);
 235
 236   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 237   ip6_address_mask (&dst_address, &im->fib_masks[dst_address_length]);
 238
 239   /* refcount accounting */
 240   if (is_del)
 241     {
 242       ASSERT (im->dst_address_length_refcounts[dst_address_length] > 0);
 243       if (--im->dst_address_length_refcounts[dst_address_length] == 0)
 244         {
 245           im->non_empty_dst_address_length_bitmap =
 246             clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 247                              128 - dst_address_length, 0);
 248           compute_prefix_lengths_in_search_order (im);
 249         }
 250     }
 251   else
 252     {
 253       im->dst_address_length_refcounts[dst_address_length]++;
 254
 255       im->non_empty_dst_address_length_bitmap =
 256         clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 257                              128 - dst_address_length, 1);
 258       compute_prefix_lengths_in_search_order (im);
 259     }
 260
 261   kv.key[0] = dst_address.as_u64[0];
 262   kv.key[1] = dst_address.as_u64[1];
 263   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 264
 265   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 266     old_adj_index = value.value;
 267
 268   if (is_del)
 269     BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 0 /* is_add */);
 270   else
 271     {
 272       /* Make sure adj index is valid. */
 273       if (CLIB_DEBUG > 0)
 274         (void) ip_get_adjacency (lm, adj_index);
 275
 276       kv.value = adj_index;
 277
 278       BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 1 /* is_add */);
 279     }
 280
 281   /* Avoid spurious reference count increments */
 282   if (old_adj_index == adj_index
 283       && adj_index != ~0
 284       && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 285     {
 286       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 287       if (adj->share_count > 0)
 288         adj->share_count --;
 289     }
 290
 291   /* Delete old adjacency index if present and changed. */
 292   {
 293     if (! (a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 294         && old_adj_index != ~0
 295         && old_adj_index != adj_index)
 296       ip_del_adjacency (lm, old_adj_index);
 297   }
 298 }
 299
 300 u32
 301 ip6_route_get_next_hop_adj (ip6_main_t * im,
 302                             u32 fib_index,
 303                             ip6_address_t *next_hop,
 304                             u32 next_hop_sw_if_index,
 305                             u32 explicit_fib_index)
 306 {
 307   ip_lookup_main_t * lm = &im->lookup_main;
 308   vnet_main_t * vnm = vnet_get_main();
 309   int is_interface_next_hop;
 310   uword * nh_result;
 311   u32 nh_adj_index;
 312   ip6_fib_t * fib;
 313
 314   fib = vec_elt_at_index (im->fibs, fib_index);
 315
 316   is_interface_next_hop = ip6_address_is_zero (next_hop);
 317
 318   if (is_interface_next_hop)
 319     {
 320       nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index,
 321                             next_hop_sw_if_index);
 322       if (nh_result)
 323           nh_adj_index = *nh_result;
 324       else
 325         {
 326           ip_adjacency_t * adj;
 327           adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 328                                   &nh_adj_index);
 329           ip6_adjacency_set_interface_route (vnm, adj,
 330                                              next_hop_sw_if_index, ~0);
 331           ip_call_add_del_adjacency_callbacks
 332               (lm, next_hop_sw_if_index, /* is_del */ 0);
 333           hash_set (im->interface_route_adj_index_by_sw_if_index,
 334                     next_hop_sw_if_index, nh_adj_index);
 335         }
 336     }
 337   else if (next_hop_sw_if_index == ~0)
 338     {
 339       /* next-hop is recursive. we always need a indirect adj
 340        * for recursive paths. Any LPM we perform now will give
 341        * us a valid adj, but without tracking the next-hop we
 342        * have no way to keep it valid.
 343        */
 344       ip_adjacency_t add_adj;
 345       memset (&add_adj, 0, sizeof(add_adj));
 346       add_adj.n_adj = 1;
 347       add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 348       add_adj.indirect.next_hop.ip6.as_u64[0] = next_hop->as_u64[0];
 349       add_adj.indirect.next_hop.ip6.as_u64[1] = next_hop->as_u64[1];
 350       add_adj.explicit_fib_index = explicit_fib_index;
 351       ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 352     }
 353   else
 354     {
 355       BVT(clib_bihash_kv) kv, value;
 356
 357       /* Look for the interface /128 route */
 358       kv.key[0] = next_hop->as_u64[0];
 359       kv.key[1] = next_hop->as_u64[1];
 360       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 361 after_nd:
 362       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 363         {
 364           ip_adjacency_t * adj;
 365           nh_adj_index = ip6_fib_lookup_with_table (im, fib_index, next_hop);
 366           adj = ip_get_adjacency (lm, nh_adj_index);
 367           /* if ND interface adjacencty is present, we need to
 368            install ND adjaceny for specific next hop */
 369           if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 370               adj->arp.next_hop.ip6.as_u64[0] == 0 &&
 371               adj->arp.next_hop.ip6.as_u64[1] == 0)
 372             {
 373               nh_adj_index = vnet_ip6_neighbor_glean_add(fib_index, next_hop);
 374             }
 375           else if (next_hop->as_u8[0] == 0xfe)
 376             {
 377               //Next hop is link-local. No indirect in this case.
 378               //Let's add it as a possible neighbor on this interface
 379               ip6_address_t null_addr= {};
 380               ip6_add_del_route_next_hop (im, IP6_ROUTE_FLAG_ADD,
 381                                           next_hop, 128,
 382                                           &null_addr, next_hop_sw_if_index,
 383                                           1, ~0, fib_index);
 384               goto after_nd;
 385             }
 386         }
 387       else
 388         {
 389           nh_adj_index = value.value;
 390         }
 391     }
 392
 393   return (nh_adj_index);
 394 }
 395
 396 void
 397 ip6_add_del_route_next_hop (ip6_main_t * im,
 398                             u32 flags,
 399                             ip6_address_t * dst_address,
 400                             u32 dst_address_length,
 401                             ip6_address_t * next_hop,
 402                             u32 next_hop_sw_if_index,
 403                             u32 next_hop_weight, u32 adj_index,
 404                             u32 explicit_fib_index)
 405 {
 406   vnet_main_t * vnm = vnet_get_main();
 407   ip_lookup_main_t * lm = &im->lookup_main;
 408   u32 fib_index;
 409   ip6_fib_t * fib;
 410   ip6_address_t masked_dst_address;
 411   u32 old_mp_adj_index, new_mp_adj_index;
 412   u32 dst_adj_index, nh_adj_index;
 413   int rv;
 414   ip_adjacency_t * dst_adj;
 415   ip_multipath_adjacency_t * old_mp, * new_mp;
 416   int is_del = (flags & IP6_ROUTE_FLAG_DEL) != 0;
 417   clib_error_t * error = 0;
 418   BVT(clib_bihash_kv) kv, value;
 419
 420   vlib_smp_unsafe_warning();
 421
 422   if (explicit_fib_index == (u32)~0)
 423     fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 424   else
 425     fib_index = explicit_fib_index;
 426
 427   fib = vec_elt_at_index (im->fibs, fib_index);
 428
 429   /* Lookup next hop to be added or deleted. */
 430   if (adj_index == (u32)~0)
 431     {
 432       nh_adj_index = ip6_route_get_next_hop_adj(im, fib_index,
 433                                                 next_hop,
 434                                                 next_hop_sw_if_index,
 435                                                 explicit_fib_index);
 436     }
 437   else
 438     {
 439       /* Look for the interface /128 route */
 440       kv.key[0] = next_hop->as_u64[0];
 441       kv.key[1] = next_hop->as_u64[1];
 442       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 443
 444       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 445         {
 446           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 447           error = clib_error_return (0, "next-hop %U/128 not in FIB",
 448                                      format_ip6_address, next_hop);
 449           goto done;
 450         }
 451
 452       nh_adj_index = value.value;
 453     }
 454
 455   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 456   masked_dst_address = dst_address[0];
 457   ip6_address_mask (&masked_dst_address, &im->fib_masks[dst_address_length]);
 458
 459   kv.key[0] = masked_dst_address.as_u64[0];
 460   kv.key[1] = masked_dst_address.as_u64[1];
 461   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 462
 463   rv = BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value);
 464
 465   if (rv == 0)
 466     {
 467       dst_adj_index = value.value;
 468       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 469     }
 470   else
 471     {
 472       /* For deletes destination must be known. */
 473       if (is_del)
 474         {
 475           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 476           error = clib_error_return (0, "unknown destination %U/%d",
 477                                      format_ip6_address, dst_address,
 478                                      dst_address_length);
 479           goto done;
 480         }
 481
 482       dst_adj_index = ~0;
 483       dst_adj = 0;
 484     }
 485
 486   /* Ignore adds of X/128 with next hop of X. */
 487   if (! is_del
 488       && dst_address_length == 128
 489       && ip6_address_is_equal (dst_address, next_hop))
 490     {
 491       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 492       error = clib_error_return (0, "prefix matches next hop %U/%d",
 493                                  format_ip6_address, dst_address,
 494                                  dst_address_length);
 495       goto done;
 496     }
 497
 498   /* Destination is not known and default weight is set so add route
 499      to existing non-multipath adjacency */
 500   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 501   {
 502     /* create / delete additional mapping of existing adjacency */
 503     ip6_add_del_route_args_t a;
 504     ip_adjacency_t * nh_adj = ip_get_adjacency (lm, nh_adj_index);
 505
 506     a.table_index_or_table_id = fib_index;
 507     a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 508         | IP6_ROUTE_FLAG_FIB_INDEX
 509         | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 510         | (flags & (IP6_ROUTE_FLAG_NO_REDISTRIBUTE
 511             | IP6_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 512     a.dst_address = dst_address[0];
 513     a.dst_address_length = dst_address_length;
 514     a.adj_index = nh_adj_index;
 515     a.add_adj = 0;
 516     a.n_add_adj = 0;
 517
 518     ip6_add_del_route (im, &a);
 519     /* adjust share count. This cannot be the only use of the adjacency */
 520     nh_adj->share_count += is_del ? -1 : 1;
 521
 522     goto done;
 523   }
 524
 525   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 526
 527   if (! ip_multipath_adjacency_add_del_next_hop
 528       (lm, is_del,
 529        dst_adj ? dst_adj->heap_handle : ~0,
 530        nh_adj_index,
 531        next_hop_weight,
 532        &new_mp_adj_index))
 533     {
 534       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 535       error = clib_error_return
 536         (0, "requested deleting next-hop %U not found in multi-path",
 537          format_ip6_address, next_hop);
 538       goto done;
 539     }
 540
 541   old_mp = new_mp = 0;
 542   if (old_mp_adj_index != ~0)
 543     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 544   if (new_mp_adj_index != ~0)
 545     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 546
 547   if (old_mp != new_mp)
 548     {
 549       ip6_add_del_route_args_t a;
 550       ip_adjacency_t * adj;
 551
 552       a.table_index_or_table_id = fib_index;
 553       a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 554                  | IP6_ROUTE_FLAG_FIB_INDEX
 555                  | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 556                  | (flags & IP6_ROUTE_FLAG_NO_REDISTRIBUTE));
 557       a.dst_address = dst_address[0];
 558       a.dst_address_length = dst_address_length;
 559       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 560       a.add_adj = 0;
 561       a.n_add_adj = 0;
 562
 563       ip6_add_del_route (im, &a);
 564
 565       adj = ip_get_adjacency (lm, new_mp ? new_mp->adj_index : dst_adj_index);
 566       if (adj->n_adj == 1)
 567         adj->share_count += is_del ? -1 : 1;
 568     }
 569
 570  done:
 571   if (error)
 572     clib_error_report (error);
 573 }
 574
 575 u32
 576 ip6_get_route (ip6_main_t * im,
 577                u32 table_index_or_table_id,
 578                u32 flags,
 579                ip6_address_t * address,
 580                u32 address_length)
 581 {
 582   ip6_fib_t * fib = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 583   ip6_address_t masked_address;
 584   BVT(clib_bihash_kv) kv, value;
 585
 586   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 587   clib_memcpy (&masked_address, address, sizeof (masked_address));
 588   ip6_address_mask (&masked_address, &im->fib_masks[address_length]);
 589
 590   kv.key[0] = masked_address.as_u64[0];
 591   kv.key[1] = masked_address.as_u64[1];
 592   kv.key[2] = ((u64)((fib - im->fibs))<<32) | address_length;
 593
 594   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 595     return (value.value);
 596   return 0;
 597 }
 598
 599 void
 600 ip6_foreach_matching_route (ip6_main_t * im,
 601                             u32 table_index_or_table_id,
 602                             u32 flags,
 603                             ip6_address_t * dst_address,
 604                             u32 address_length,
 605                             ip6_address_t ** results,
 606                             u8 ** result_lengths)
 607 {
 608   ip6_fib_t * fib =
 609     find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 610   BVT(clib_bihash) * h = &im->ip6_lookup_table;
 611   BVT(clib_bihash_value) * v;
 612   clib_bihash_bucket_t * b;
 613   int i, j, k;
 614
 615   if (*results)
 616     _vec_len (*results) = 0;
 617   if (*result_lengths)
 618     _vec_len (*result_lengths) = 0;
 619
 620   /* Walk the table looking for routes which match the supplied address */
 621   for (i = 0; i < h->nbuckets; i++)
 622     {
 623       b = &h->buckets [i];
 624       if (b->offset == 0)
 625           continue;
 626
 627       v = BV(clib_bihash_get_value) (h, b->offset);
 628       for (j = 0; j < (1<<b->log2_pages); j++)
 629         {
 630           for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
 631             {
 632               if (BV(clib_bihash_is_free)(&v->kvp[k]))
 633                 continue;
 634
 635               if ((v->kvp[k].key[2]
 636                    == (((u64)((fib - im->fibs))<<32) | address_length))
 637                   && ip6_destination_matches_route
 638                   (im, dst_address, (ip6_address_t *) &v->kvp[k],
 639                    address_length))
 640                 {
 641                   ip6_address_t * a;
 642
 643                   a = (ip6_address_t *)(&v->kvp[k]);
 644
 645                   vec_add1 (*results, a[0]);
 646                   vec_add1 (*result_lengths, address_length);
 647                 }
 648             }
 649           v++;
 650         }
 651     }
 652 }
 653
 654 void ip6_maybe_remap_adjacencies (ip6_main_t * im,
 655                                   u32 table_index_or_table_id,
 656                                   u32 flags)
 657 {
 658 #if SOONE
 659   ip6_fib_t * fib
 660     = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 661 #endif
 662   ip_lookup_main_t * lm = &im->lookup_main;
 663
 664   if (lm->n_adjacency_remaps == 0)
 665     return;
 666
 667   clib_warning ("unimplemented, please report to vpp-dev@cisco.com");
 668
 669   /* All remaps have been performed. */
 670   lm->n_adjacency_remaps = 0;
 671 }
 672
 673 void ip6_delete_matching_routes (ip6_main_t * im,
 674                                  u32 table_index_or_table_id,
 675                                  u32 flags,
 676                                  ip6_address_t * address,
 677                                  u32 address_length)
 678 {
 679   /* $$$$ static may be OK - this should happen only on thread 0 */
 680   static ip6_address_t * matching_addresses;
 681   static u8 * matching_address_lengths;
 682   u32 l, i;
 683   ip6_add_del_route_args_t a;
 684
 685   vlib_smp_unsafe_warning();
 686
 687   a.flags = IP6_ROUTE_FLAG_DEL | IP6_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 688   a.table_index_or_table_id = table_index_or_table_id;
 689   a.adj_index = ~0;
 690   a.add_adj = 0;
 691   a.n_add_adj = 0;
 692
 693   for (l = address_length + 1; l <= 128; l++)
 694     {
 695       ip6_foreach_matching_route (im, table_index_or_table_id, flags,
 696                                   address,
 697                                   l,
 698                                   &matching_addresses,
 699                                   &matching_address_lengths);
 700       for (i = 0; i < vec_len (matching_addresses); i++)
 701         {
 702           a.dst_address = matching_addresses[i];
 703           a.dst_address_length = matching_address_lengths[i];
 704           ip6_add_del_route (im, &a);
 705         }
 706     }
 707
 708   ip6_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 709 }
 710
 711 void
 712 ip6_forward_next_trace (vlib_main_t * vm,
 713                         vlib_node_runtime_t * node,
 714                         vlib_frame_t * frame,
 715                         vlib_rx_or_tx_t which_adj_index);
 716
 717 always_inline uword
 718 ip6_lookup_inline (vlib_main_t * vm,
 719                    vlib_node_runtime_t * node,
 720                    vlib_frame_t * frame,
 721                    int is_indirect)
 722 {
 723   ip6_main_t * im = &ip6_main;
 724   ip_lookup_main_t * lm = &im->lookup_main;
 725   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 726   u32 n_left_from, n_left_to_next, * from, * to_next;
 727   ip_lookup_next_t next;
 728   u32 cpu_index = os_get_cpu_number();
 729
 730   from = vlib_frame_vector_args (frame);
 731   n_left_from = frame->n_vectors;
 732   next = node->cached_next_index;
 733
 734   while (n_left_from > 0)
 735     {
 736       vlib_get_next_frame (vm, node, next,
 737                            to_next, n_left_to_next);
 738
 739       while (n_left_from >= 4 && n_left_to_next >= 2)
 740         {
 741           vlib_buffer_t * p0, * p1;
 742           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
 743           ip_lookup_next_t next0, next1;
 744           ip6_header_t * ip0, * ip1;
 745           ip_adjacency_t * adj0, * adj1;
 746           ip6_address_t * dst_addr0, * dst_addr1;
 747           u32 fib_index0, fib_index1;
 748           u32 flow_hash_config0, flow_hash_config1;
 749
 750           /* Prefetch next iteration. */
 751           {
 752             vlib_buffer_t * p2, * p3;
 753
 754             p2 = vlib_get_buffer (vm, from[2]);
 755             p3 = vlib_get_buffer (vm, from[3]);
 756
 757             vlib_prefetch_buffer_header (p2, LOAD);
 758             vlib_prefetch_buffer_header (p3, LOAD);
 759             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 760             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 761           }
 762
 763           pi0 = to_next[0] = from[0];
 764           pi1 = to_next[1] = from[1];
 765
 766           p0 = vlib_get_buffer (vm, pi0);
 767           p1 = vlib_get_buffer (vm, pi1);
 768
 769           ip0 = vlib_buffer_get_current (p0);
 770           ip1 = vlib_buffer_get_current (p1);
 771
 772           if (is_indirect)
 773             {
 774               ip_adjacency_t * iadj0, * iadj1;
 775               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 776               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 777               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 778               dst_addr1 = &iadj1->indirect.next_hop.ip6;
 779             }
 780           else
 781             {
 782               dst_addr0 = &ip0->dst_address;
 783               dst_addr1 = &ip1->dst_address;
 784             }
 785
 786           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 787           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 788
 789           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 790             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 791           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 792             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 793
 794           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 795           adj_index1 = ip6_fib_lookup_with_table (im, fib_index1, dst_addr1);
 796
 797           adj0 = ip_get_adjacency (lm, adj_index0);
 798           adj1 = ip_get_adjacency (lm, adj_index1);
 799
 800           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 801             {
 802               adj_index0 = ip6_fib_lookup_with_table
 803                 (im, adj0->explicit_fib_index, dst_addr0);
 804               adj0 = ip_get_adjacency (lm, adj_index0);
 805             }
 806           if (PREDICT_FALSE (adj1->explicit_fib_index != ~0))
 807             {
 808               adj_index1 = ip6_fib_lookup_with_table
 809                 (im, adj1->explicit_fib_index, dst_addr1);
 810               adj1 = ip_get_adjacency (lm, adj_index1);
 811             }
 812
 813           next0 = adj0->lookup_next_index;
 814           next1 = adj1->lookup_next_index;
 815
 816           /* Only process the HBH Option Header if explicitly configured to do so */
 817           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 818             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 819           next1 = (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 820             adj_index1 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj1->lookup_next_index;
 821
 822           vnet_buffer (p0)->ip.flow_hash =
 823             vnet_buffer(p1)->ip.flow_hash = 0;
 824
 825           if (PREDICT_FALSE(adj0->n_adj > 1))
 826             {
 827               flow_hash_config0 =
 828                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 829               vnet_buffer (p0)->ip.flow_hash =
 830                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 831             }
 832
 833           if (PREDICT_FALSE(adj1->n_adj > 1))
 834             {
 835               flow_hash_config1 =
 836                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 837
 838               vnet_buffer (p1)->ip.flow_hash =
 839                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 840             }
 841
 842           ASSERT (adj0->n_adj > 0);
 843           ASSERT (adj1->n_adj > 0);
 844           ASSERT (is_pow2 (adj0->n_adj));
 845           ASSERT (is_pow2 (adj1->n_adj));
 846           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 847           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
 848
 849           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 850           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 851
 852           vlib_increment_combined_counter
 853               (cm, cpu_index, adj_index0, 1,
 854                vlib_buffer_length_in_chain (vm, p0));
 855           vlib_increment_combined_counter
 856               (cm, cpu_index, adj_index1, 1,
 857                vlib_buffer_length_in_chain (vm, p1));
 858
 859           from += 2;
 860           to_next += 2;
 861           n_left_to_next -= 2;
 862           n_left_from -= 2;
 863
 864           wrong_next = (next0 != next) + 2*(next1 != next);
 865           if (PREDICT_FALSE (wrong_next != 0))
 866             {
 867               switch (wrong_next)
 868                 {
 869                 case 1:
 870                   /* A B A */
 871                   to_next[-2] = pi1;
 872                   to_next -= 1;
 873                   n_left_to_next += 1;
 874                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 875                   break;
 876
 877                 case 2:
 878                   /* A A B */
 879                   to_next -= 1;
 880                   n_left_to_next += 1;
 881                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 882                   break;
 883
 884                 case 3:
 885                   /* A B C */
 886                   to_next -= 2;
 887                   n_left_to_next += 2;
 888                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 889                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 890                   if (next0 == next1)
 891                     {
 892                       /* A B B */
 893                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 894                       next = next1;
 895                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 896                     }
 897                 }
 898             }
 899         }
 900
 901       while (n_left_from > 0 && n_left_to_next > 0)
 902         {
 903           vlib_buffer_t * p0;
 904           ip6_header_t * ip0;
 905           u32 pi0, adj_index0;
 906           ip_lookup_next_t next0;
 907           ip_adjacency_t * adj0;
 908           ip6_address_t * dst_addr0;
 909           u32 fib_index0, flow_hash_config0;
 910
 911           pi0 = from[0];
 912           to_next[0] = pi0;
 913
 914           p0 = vlib_get_buffer (vm, pi0);
 915
 916           ip0 = vlib_buffer_get_current (p0);
 917
 918           if (is_indirect)
 919             {
 920               ip_adjacency_t * iadj0;
 921               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 922               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 923             }
 924           else
 925             {
 926               dst_addr0 = &ip0->dst_address;
 927             }
 928
 929           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 930           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 931             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 932
 933           flow_hash_config0 =
 934               vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 935
 936           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 937
 938           adj0 = ip_get_adjacency (lm, adj_index0);
 939
 940           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 941             {
 942               adj_index0 = ip6_fib_lookup_with_table
 943                 (im, adj0->explicit_fib_index, dst_addr0);
 944               adj0 = ip_get_adjacency (lm, adj_index0);
 945             }
 946
 947           /* Only process the HBH Option Header if explicitly configured to do so */
 948           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 949             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 950
 951           vnet_buffer (p0)->ip.flow_hash = 0;
 952
 953           if (PREDICT_FALSE(adj0->n_adj > 1))
 954             {
 955               flow_hash_config0 =
 956                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 957               vnet_buffer (p0)->ip.flow_hash =
 958                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 959             }
 960
 961           ASSERT (adj0->n_adj > 0);
 962           ASSERT (is_pow2 (adj0->n_adj));
 963           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 964
 965           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 966
 967           vlib_increment_combined_counter
 968               (cm, cpu_index, adj_index0, 1,
 969                vlib_buffer_length_in_chain (vm, p0));
 970
 971           from += 1;
 972           to_next += 1;
 973           n_left_to_next -= 1;
 974           n_left_from -= 1;
 975
 976           if (PREDICT_FALSE (next0 != next))
 977             {
 978               n_left_to_next += 1;
 979               vlib_put_next_frame (vm, node, next, n_left_to_next);
 980               next = next0;
 981               vlib_get_next_frame (vm, node, next,
 982                                    to_next, n_left_to_next);
 983               to_next[0] = pi0;
 984               to_next += 1;
 985               n_left_to_next -= 1;
 986             }
 987         }
 988
 989       vlib_put_next_frame (vm, node, next, n_left_to_next);
 990     }
 991
 992   if (node->flags & VLIB_NODE_FLAG_TRACE)
 993       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 994
 995   return frame->n_vectors;
 996 }
 997
 998 void ip6_adjacency_set_interface_route (vnet_main_t * vnm,
 999                                         ip_adjacency_t * adj,
1000                                         u32 sw_if_index,
1001                                         u32 if_address_index)
1002 {
1003   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
1004   ip_lookup_next_t n;
1005   u32 node_index;
1006
1007   if (hw->hw_class_index == ethernet_hw_interface_class.index
1008       || hw->hw_class_index == srp_hw_interface_class.index)
1009     {
1010       n = IP_LOOKUP_NEXT_ARP;
1011       node_index = ip6_discover_neighbor_node.index;
1012       adj->if_address_index = if_address_index;
1013       adj->arp.next_hop.ip6.as_u64[0] = 0;
1014       adj->arp.next_hop.ip6.as_u64[1] = 0;
1015   }
1016   else
1017     {
1018       n = IP_LOOKUP_NEXT_REWRITE;
1019       node_index = ip6_rewrite_node.index;
1020     }
1021
1022  adj->lookup_next_index = n;
1023  adj->explicit_fib_index = ~0;
1024
1025  vnet_rewrite_for_sw_interface
1026    (vnm,
1027     VNET_L3_PACKET_TYPE_IP6,
1028     sw_if_index,
1029     node_index,
1030     VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
1031     &adj->rewrite_header,
1032     sizeof (adj->rewrite_data));
1033 }
1034
1035 static void
1036 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
1037                           ip6_main_t * im, u32 fib_index,
1038                           ip_interface_address_t * a)
1039 {
1040   ip_lookup_main_t * lm = &im->lookup_main;
1041   ip_adjacency_t * adj;
1042   ip6_address_t * address = ip_interface_address_get_address (lm, a);
1043   ip6_add_del_route_args_t x;
1044   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1045   u32 classify_table_index;
1046
1047   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1048   x.table_index_or_table_id = fib_index;
1049   x.flags = (IP6_ROUTE_FLAG_ADD
1050              | IP6_ROUTE_FLAG_FIB_INDEX
1051              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1052   x.dst_address = address[0];
1053   x.dst_address_length = a->address_length;
1054   x.n_add_adj = 0;
1055   x.add_adj = 0;
1056
1057   a->neighbor_probe_adj_index = ~0;
1058   if (a->address_length < 128)
1059     {
1060       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1061                               &x.adj_index);
1062       ip6_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1063       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1064       ip6_add_del_route (im, &x);
1065       a->neighbor_probe_adj_index = x.adj_index;
1066     }
1067
1068   /* Add e.g. ::1/128 as local to this host. */
1069   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1070                           &x.adj_index);
1071
1072   classify_table_index = ~0;
1073   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1074     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1075   if (classify_table_index != (u32) ~0)
1076     {
1077       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1078       adj->classify.table_index = classify_table_index;
1079     }
1080   else
1081     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1082
1083   adj->if_address_index = a - lm->if_address_pool;
1084   adj->rewrite_header.sw_if_index = sw_if_index;
1085   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1086   adj->rewrite_header.data_bytes = 0;
1087   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1088   x.dst_address_length = 128;
1089   ip6_add_del_route (im, &x);
1090 }
1091
1092 static void
1093 ip6_del_interface_routes (ip6_main_t * im, u32 fib_index,
1094                           ip6_address_t * address, u32 address_length)
1095 {
1096   ip6_add_del_route_args_t x;
1097
1098   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1099   x.table_index_or_table_id = fib_index;
1100   x.flags = (IP6_ROUTE_FLAG_DEL
1101              | IP6_ROUTE_FLAG_FIB_INDEX
1102              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1103   x.dst_address = address[0];
1104   x.dst_address_length = address_length;
1105   x.adj_index = ~0;
1106   x.n_add_adj = 0;
1107   x.add_adj = 0;
1108
1109   if (address_length < 128)
1110     {
1111       /* Don't wipe out fe80::0/64 */
1112       if (address_length != 64 ||
1113           address[0].as_u64[0] != clib_net_to_host_u64(0xfe80000000000000ULL))
1114         ip6_add_del_route (im, &x);
1115     }
1116
1117   x.dst_address_length = 128;
1118   ip6_add_del_route (im, &x);
1119
1120   ip6_delete_matching_routes (im,
1121                               fib_index,
1122                               IP6_ROUTE_FLAG_FIB_INDEX,
1123                               address,
1124                               address_length);
1125 }
1126
1127 typedef struct {
1128     u32 sw_if_index;
1129     ip6_address_t address;
1130     u32 length;
1131 } ip6_interface_address_t;
1132
1133 static clib_error_t *
1134 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1135                                         u32 sw_if_index,
1136                                         ip6_address_t * new_address,
1137                                         u32 new_length,
1138                                         u32 redistribute,
1139                                         u32 insert_routes,
1140                                         u32 is_del);
1141
1142 static clib_error_t *
1143 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1144                                         u32 sw_if_index,
1145                                         ip6_address_t * address,
1146                                         u32 address_length,
1147                                         u32 redistribute,
1148                                         u32 insert_routes,
1149                                         u32 is_del)
1150 {
1151   vnet_main_t * vnm = vnet_get_main();
1152   ip6_main_t * im = &ip6_main;
1153   ip_lookup_main_t * lm = &im->lookup_main;
1154   clib_error_t * error;
1155   u32 if_address_index;
1156   ip6_address_fib_t ip6_af, * addr_fib = 0;
1157
1158   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1159   ip6_addr_fib_init (&ip6_af, address,
1160                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1161   vec_add1 (addr_fib, ip6_af);
1162
1163   {
1164     uword elts_before = pool_elts (lm->if_address_pool);
1165
1166     error = ip_interface_address_add_del
1167       (lm,
1168        sw_if_index,
1169        addr_fib,
1170        address_length,
1171        is_del,
1172        &if_address_index);
1173     if (error)
1174       goto done;
1175
1176     /* Pool did not grow: add duplicate address. */
1177     if (elts_before == pool_elts (lm->if_address_pool))
1178       goto done;
1179   }
1180
1181   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1182     {
1183       if (is_del)
1184         ip6_del_interface_routes (im, ip6_af.fib_index, address,
1185                                   address_length);
1186
1187       else
1188         ip6_add_interface_routes (vnm, sw_if_index,
1189                                   im, ip6_af.fib_index,
1190                                   pool_elt_at_index (lm->if_address_pool, if_address_index));
1191     }
1192
1193   {
1194     ip6_add_del_interface_address_callback_t * cb;
1195     vec_foreach (cb, im->add_del_interface_address_callbacks)
1196       cb->function (im, cb->function_opaque, sw_if_index,
1197                     address, address_length,
1198                     if_address_index,
1199                     is_del);
1200   }
1201
1202  done:
1203   vec_free (addr_fib);
1204   return error;
1205 }
1206
1207 clib_error_t *
1208 ip6_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1209                                ip6_address_t * address, u32 address_length,
1210                                u32 is_del)
1211 {
1212   return ip6_add_del_interface_address_internal
1213     (vm, sw_if_index, address, address_length,
1214      /* redistribute */ 1,
1215      /* insert_routes */ 1,
1216      is_del);
1217 }
1218
1219 clib_error_t *
1220 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
1221                                 u32 sw_if_index,
1222                                 u32 flags)
1223 {
1224   ip6_main_t * im = &ip6_main;
1225   ip_interface_address_t * ia;
1226   ip6_address_t * a;
1227   u32 is_admin_up, fib_index;
1228
1229   /* Fill in lookup tables with default table (0). */
1230   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1231
1232   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1233
1234   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1235
1236   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1237
1238   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1239                                 0 /* honor unnumbered */,
1240   ({
1241     a = ip_interface_address_get_address (&im->lookup_main, ia);
1242     if (is_admin_up)
1243       ip6_add_interface_routes (vnm, sw_if_index,
1244                                 im, fib_index,
1245                                 ia);
1246     else
1247       ip6_del_interface_routes (im, fib_index,
1248                                 a, ia->address_length);
1249   }));
1250
1251   return 0;
1252 }
1253
1254 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
1255
1256 /* Built-in ip6 unicast rx feature path definition */
1257 VNET_IP6_UNICAST_FEATURE_INIT (ip6_inacl, static) = {
1258   .node_name = "ip6-inacl",
1259   .runs_before = {"ipsec-input-ip6", 0},
1260   .feature_index = &ip6_main.ip6_unicast_rx_feature_check_access,
1261 };
1262
1263 VNET_IP6_UNICAST_FEATURE_INIT (ip6_ipsec, static) = {
1264   .node_name = "ipsec-input-ip6",
1265   .runs_before = {"l2tp-decap", 0},
1266   .feature_index = &ip6_main.ip6_unicast_rx_feature_ipsec,
1267 };
1268
1269 VNET_IP6_UNICAST_FEATURE_INIT (ip6_l2tp, static) = {
1270   .node_name = "l2tp-decap",
1271   .runs_before = {"vpath-input-ip6", 0},
1272   .feature_index = &ip6_main.ip6_unicast_rx_feature_l2tp_decap,
1273 };
1274
1275 VNET_IP6_UNICAST_FEATURE_INIT (ip6_vpath, static) = {
1276   .node_name = "vpath-input-ip6",
1277   .runs_before = {"ip6-lookup", 0},
1278   .feature_index = &ip6_main.ip6_unicast_rx_feature_vpath,
1279 };
1280
1281 VNET_IP6_UNICAST_FEATURE_INIT (ip6_lookup, static) = {
1282   .node_name = "ip6-lookup",
1283   .runs_before = {0}, /* not before any other features */
1284   .feature_index = &ip6_main.ip6_unicast_rx_feature_lookup,
1285 };
1286
1287 /* Built-in ip6 multicast rx feature path definition (none now) */
1288 VNET_IP6_MULTICAST_FEATURE_INIT (ip4_vpath_mc, static) = {
1289   .node_name = "vpath-input-ip6",
1290   .runs_before = {"ip6-lookup", 0},
1291   .feature_index = &ip6_main.ip6_multicast_rx_feature_vpath,
1292 };
1293
1294 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_lookup, static) = {
1295   .node_name = "ip6-lookup",
1296   .runs_before = {0}, /* not before any other features */
1297   .feature_index = &ip6_main.ip6_multicast_rx_feature_lookup,
1298 };
1299
1300 static char * feature_start_nodes[] =
1301   {"ip6-input"};
1302
1303 static clib_error_t *
1304 ip6_feature_init (vlib_main_t * vm, ip6_main_t * im)
1305 {
1306   ip_lookup_main_t * lm = &im->lookup_main;
1307   clib_error_t * error;
1308   vnet_cast_t cast;
1309
1310   for (cast = 0; cast < VNET_N_CAST; cast++)
1311     {
1312       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1313       vnet_config_main_t * vcm = &cm->config_main;
1314
1315       if ((error = ip_feature_init_cast (vm, cm, vcm,
1316                                          feature_start_nodes,
1317                                          ARRAY_LEN(feature_start_nodes),
1318                                          cast,
1319                                          0 /* is_ip4 */)))
1320         return error;
1321     }
1322   return 0;
1323 }
1324
1325 clib_error_t *
1326 ip6_sw_interface_add_del (vnet_main_t * vnm,
1327                           u32 sw_if_index,
1328                           u32 is_add)
1329 {
1330   vlib_main_t * vm = vnm->vlib_main;
1331   ip6_main_t * im = &ip6_main;
1332   ip_lookup_main_t * lm = &im->lookup_main;
1333   u32 ci, cast;
1334   u32 feature_index;
1335
1336   for (cast = 0; cast < VNET_N_CAST; cast++)
1337     {
1338       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1339       vnet_config_main_t * vcm = &cm->config_main;
1340
1341       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1342       ci = cm->config_index_by_sw_if_index[sw_if_index];
1343
1344       if (cast == VNET_UNICAST)
1345         feature_index = im->ip6_unicast_rx_feature_lookup;
1346       else
1347         feature_index = im->ip6_multicast_rx_feature_lookup;
1348
1349       if (is_add)
1350         ci = vnet_config_add_feature (vm, vcm,
1351                                       ci,
1352                                       feature_index,
1353                                       /* config data */ 0,
1354                                       /* # bytes of config data */ 0);
1355       else
1356         ci = vnet_config_del_feature (vm, vcm,
1357                                       ci,
1358                                       feature_index,
1359                                       /* config data */ 0,
1360                                       /* # bytes of config data */ 0);
1361
1362       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1363     }
1364   return /* no error */ 0;
1365 }
1366
1367 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
1368
1369 static uword
1370 ip6_lookup (vlib_main_t * vm,
1371             vlib_node_runtime_t * node,
1372             vlib_frame_t * frame)
1373 {
1374   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 0);
1375 }
1376
1377 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
1378
1379 VLIB_REGISTER_NODE (ip6_lookup_node) = {
1380   .function = ip6_lookup,
1381   .name = "ip6-lookup",
1382   .vector_size = sizeof (u32),
1383
1384   .format_trace = format_ip6_lookup_trace,
1385
1386   .n_next_nodes = IP6_LOOKUP_N_NEXT,
1387   .next_nodes = IP6_LOOKUP_NEXT_NODES,
1388 };
1389
1390 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
1391
1392 static uword
1393 ip6_indirect (vlib_main_t * vm,
1394               vlib_node_runtime_t * node,
1395               vlib_frame_t * frame)
1396 {
1397   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 1);
1398 }
1399
1400
1401 VLIB_REGISTER_NODE (ip6_indirect_node) = {
1402   .function = ip6_indirect,
1403   .name = "ip6-indirect",
1404   .vector_size = sizeof (u32),
1405   .sibling_of = "ip6-lookup",
1406   .format_trace = format_ip6_lookup_trace,
1407   .n_next_nodes = 0,
1408 };
1409
1410 VLIB_NODE_FUNCTION_MULTIARCH (ip6_indirect_node, ip6_indirect)
1411
1412 typedef struct {
1413   /* Adjacency taken. */
1414   u32 adj_index;
1415   u32 flow_hash;
1416   u32 fib_index;
1417
1418   /* Packet data, possibly *after* rewrite. */
1419   u8 packet_data[128 - 1*sizeof(u32)];
1420 } ip6_forward_next_trace_t;
1421
1422 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
1423 {
1424   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1425   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1426   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1427   uword indent = format_get_indent (s);
1428
1429   s = format(s, "%U%U",
1430              format_white_space, indent,
1431              format_ip6_header, t->packet_data);
1432   return s;
1433 }
1434
1435 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
1436 {
1437   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1438   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1439   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1440   vnet_main_t * vnm = vnet_get_main();
1441   ip6_main_t * im = &ip6_main;
1442   uword indent = format_get_indent (s);
1443
1444   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1445               t->fib_index, t->adj_index, format_ip_adjacency,
1446               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1447   s = format(s, "\n%U%U",
1448              format_white_space, indent,
1449              format_ip6_header, t->packet_data);
1450   return s;
1451 }
1452
1453
1454 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
1455 {
1456   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1457   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1458   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1459   vnet_main_t * vnm = vnet_get_main();
1460   ip6_main_t * im = &ip6_main;
1461   uword indent = format_get_indent (s);
1462
1463   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1464               t->fib_index, t->adj_index, format_ip_adjacency,
1465               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1466   s = format (s, "\n%U%U",
1467               format_white_space, indent,
1468               format_ip_adjacency_packet_data,
1469               vnm, &im->lookup_main, t->adj_index,
1470               t->packet_data, sizeof (t->packet_data));
1471   return s;
1472 }
1473
1474 /* Common trace function for all ip6-forward next nodes. */
1475 void
1476 ip6_forward_next_trace (vlib_main_t * vm,
1477                         vlib_node_runtime_t * node,
1478                         vlib_frame_t * frame,
1479                         vlib_rx_or_tx_t which_adj_index)
1480 {
1481   u32 * from, n_left;
1482   ip6_main_t * im = &ip6_main;
1483
1484   n_left = frame->n_vectors;
1485   from = vlib_frame_vector_args (frame);
1486
1487   while (n_left >= 4)
1488     {
1489       u32 bi0, bi1;
1490       vlib_buffer_t * b0, * b1;
1491       ip6_forward_next_trace_t * t0, * t1;
1492
1493       /* Prefetch next iteration. */
1494       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1495       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1496
1497       bi0 = from[0];
1498       bi1 = from[1];
1499
1500       b0 = vlib_get_buffer (vm, bi0);
1501       b1 = vlib_get_buffer (vm, bi1);
1502
1503       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1504         {
1505           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1506           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1507           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1508           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1509               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1510               vec_elt (im->fib_index_by_sw_if_index,
1511                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1512
1513           clib_memcpy (t0->packet_data,
1514                   vlib_buffer_get_current (b0),
1515                   sizeof (t0->packet_data));
1516         }
1517       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1518         {
1519           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1520           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1521           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1522           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1523               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1524               vec_elt (im->fib_index_by_sw_if_index,
1525                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1526
1527           clib_memcpy (t1->packet_data,
1528                   vlib_buffer_get_current (b1),
1529                   sizeof (t1->packet_data));
1530         }
1531       from += 2;
1532       n_left -= 2;
1533     }
1534
1535   while (n_left >= 1)
1536     {
1537       u32 bi0;
1538       vlib_buffer_t * b0;
1539       ip6_forward_next_trace_t * t0;
1540
1541       bi0 = from[0];
1542
1543       b0 = vlib_get_buffer (vm, bi0);
1544
1545       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1546         {
1547           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1548           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1549           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1550           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1551               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1552               vec_elt (im->fib_index_by_sw_if_index,
1553                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1554
1555           clib_memcpy (t0->packet_data,
1556                   vlib_buffer_get_current (b0),
1557                   sizeof (t0->packet_data));
1558         }
1559       from += 1;
1560       n_left -= 1;
1561     }
1562 }
1563
1564 static uword
1565 ip6_drop_or_punt (vlib_main_t * vm,
1566                   vlib_node_runtime_t * node,
1567                   vlib_frame_t * frame,
1568                   ip6_error_t error_code)
1569 {
1570   u32 * buffers = vlib_frame_vector_args (frame);
1571   uword n_packets = frame->n_vectors;
1572
1573   vlib_error_drop_buffers (vm, node,
1574                            buffers,
1575                            /* stride */ 1,
1576                            n_packets,
1577                            /* next */ 0,
1578                            ip6_input_node.index,
1579                            error_code);
1580
1581   if (node->flags & VLIB_NODE_FLAG_TRACE)
1582     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1583
1584   return n_packets;
1585 }
1586
1587 static uword
1588 ip6_drop (vlib_main_t * vm,
1589           vlib_node_runtime_t * node,
1590           vlib_frame_t * frame)
1591 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1592
1593 static uword
1594 ip6_punt (vlib_main_t * vm,
1595           vlib_node_runtime_t * node,
1596           vlib_frame_t * frame)
1597 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1598
1599 static uword
1600 ip6_miss (vlib_main_t * vm,
1601           vlib_node_runtime_t * node,
1602           vlib_frame_t * frame)
1603 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_DST_LOOKUP_MISS); }
1604
1605 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1606   .function = ip6_drop,
1607   .name = "ip6-drop",
1608   .vector_size = sizeof (u32),
1609
1610   .format_trace = format_ip6_forward_next_trace,
1611
1612   .n_next_nodes = 1,
1613   .next_nodes = {
1614     [0] = "error-drop",
1615   },
1616 };
1617
1618 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1619
1620 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1621   .function = ip6_punt,
1622   .name = "ip6-punt",
1623   .vector_size = sizeof (u32),
1624
1625   .format_trace = format_ip6_forward_next_trace,
1626
1627   .n_next_nodes = 1,
1628   .next_nodes = {
1629     [0] = "error-punt",
1630   },
1631 };
1632
1633 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1634
1635 VLIB_REGISTER_NODE (ip6_miss_node,static) = {
1636   .function = ip6_miss,
1637   .name = "ip6-miss",
1638   .vector_size = sizeof (u32),
1639
1640   .format_trace = format_ip6_forward_next_trace,
1641
1642   .n_next_nodes = 1,
1643   .next_nodes = {
1644     [0] = "error-drop",
1645   },
1646 };
1647
1648 VLIB_NODE_FUNCTION_MULTIARCH (ip6_miss_node, ip6_miss)
1649
1650 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1651   .function = ip6_drop,
1652   .name = "ip6-multicast",
1653   .vector_size = sizeof (u32),
1654
1655   .format_trace = format_ip6_forward_next_trace,
1656
1657   .n_next_nodes = 1,
1658   .next_nodes = {
1659     [0] = "error-drop",
1660   },
1661 };
1662
1663 /* Compute TCP/UDP/ICMP6 checksum in software. */
1664 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1665 {
1666   ip_csum_t sum0;
1667   u16 sum16, payload_length_host_byte_order;
1668   u32 i, n_this_buffer, n_bytes_left;
1669   u32 headers_size = sizeof(ip0[0]);
1670   void * data_this_buffer;
1671
1672   ASSERT(bogus_lengthp);
1673   *bogus_lengthp = 0;
1674
1675   /* Initialize checksum with ip header. */
1676   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1677   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1678   data_this_buffer = (void *) (ip0 + 1);
1679
1680   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1681     {
1682       sum0 = ip_csum_with_carry (sum0,
1683                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1684       sum0 = ip_csum_with_carry (sum0,
1685                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1686     }
1687
1688   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1689   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1690     {
1691       u32  skip_bytes;
1692       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1693
1694       /* validate really icmp6 next */
1695       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1696
1697       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1698       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1699
1700       payload_length_host_byte_order  -= skip_bytes;
1701       headers_size += skip_bytes;
1702    }
1703
1704   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1705 #if DPDK > 0
1706   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1707   {
1708     struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
1709     u8 nb_segs = mb->nb_segs;
1710
1711     n_this_buffer = (p0->current_length > headers_size ?
1712                      p0->current_length - headers_size : 0);
1713     while (n_bytes_left)
1714       {
1715         sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1716         n_bytes_left -= n_this_buffer;
1717
1718         mb = mb->next;
1719         nb_segs--;
1720         if ((nb_segs == 0) || (mb == 0))
1721           break;
1722
1723         data_this_buffer = rte_ctrlmbuf_data(mb);
1724         n_this_buffer = mb->data_len;
1725       }
1726     if (n_bytes_left || nb_segs)
1727       {
1728         *bogus_lengthp = 1;
1729         return 0xfefe;
1730       }
1731   }
1732   else sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1733 #else
1734   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1735     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1736   while (1)
1737     {
1738       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1739       n_bytes_left -= n_this_buffer;
1740       if (n_bytes_left == 0)
1741         break;
1742
1743       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1744         {
1745           *bogus_lengthp = 1;
1746           return 0xfefe;
1747         }
1748       p0 = vlib_get_buffer (vm, p0->next_buffer);
1749       data_this_buffer = vlib_buffer_get_current (p0);
1750       n_this_buffer = p0->current_length;
1751     }
1752 #endif /* DPDK */
1753
1754   sum16 = ~ ip_csum_fold (sum0);
1755
1756   return sum16;
1757 }
1758
1759 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1760 {
1761   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1762   udp_header_t * udp0;
1763   u16 sum16;
1764   int bogus_length;
1765
1766   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1767   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1768           || ip0->protocol == IP_PROTOCOL_ICMP6
1769           || ip0->protocol == IP_PROTOCOL_UDP
1770           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1771
1772   udp0 = (void *) (ip0 + 1);
1773   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1774     {
1775       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1776                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1777       return p0->flags;
1778     }
1779
1780   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1781
1782   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1783                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1784
1785   return p0->flags;
1786 }
1787
1788 static uword
1789 ip6_local (vlib_main_t * vm,
1790            vlib_node_runtime_t * node,
1791            vlib_frame_t * frame)
1792 {
1793   ip6_main_t * im = &ip6_main;
1794   ip_lookup_main_t * lm = &im->lookup_main;
1795   ip_local_next_t next_index;
1796   u32 * from, * to_next, n_left_from, n_left_to_next;
1797   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1798
1799   from = vlib_frame_vector_args (frame);
1800   n_left_from = frame->n_vectors;
1801   next_index = node->cached_next_index;
1802
1803   if (node->flags & VLIB_NODE_FLAG_TRACE)
1804     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1805
1806   while (n_left_from > 0)
1807     {
1808       vlib_get_next_frame (vm, node, next_index,
1809                            to_next, n_left_to_next);
1810
1811       while (n_left_from >= 4 && n_left_to_next >= 2)
1812         {
1813           vlib_buffer_t * p0, * p1;
1814           ip6_header_t * ip0, * ip1;
1815           udp_header_t * udp0, * udp1;
1816           u32 pi0, ip_len0, udp_len0, flags0, next0;
1817           u32 pi1, ip_len1, udp_len1, flags1, next1;
1818           i32 len_diff0, len_diff1;
1819           u8 error0, type0, good_l4_checksum0;
1820           u8 error1, type1, good_l4_checksum1;
1821
1822           pi0 = to_next[0] = from[0];
1823           pi1 = to_next[1] = from[1];
1824           from += 2;
1825           n_left_from -= 2;
1826           to_next += 2;
1827           n_left_to_next -= 2;
1828
1829           p0 = vlib_get_buffer (vm, pi0);
1830           p1 = vlib_get_buffer (vm, pi1);
1831
1832           ip0 = vlib_buffer_get_current (p0);
1833           ip1 = vlib_buffer_get_current (p1);
1834
1835           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1836           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1837
1838           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1839           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1840
1841           flags0 = p0->flags;
1842           flags1 = p1->flags;
1843
1844           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1845           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1846
1847           udp0 = ip6_next_header (ip0);
1848           udp1 = ip6_next_header (ip1);
1849
1850           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1851           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1852           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1853
1854           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1855           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1856
1857           /* Verify UDP length. */
1858           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1859           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1860           udp_len0 = clib_net_to_host_u16 (udp0->length);
1861           udp_len1 = clib_net_to_host_u16 (udp1->length);
1862
1863           len_diff0 = ip_len0 - udp_len0;
1864           len_diff1 = ip_len1 - udp_len1;
1865
1866           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1867           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1868
1869           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1870                              && ! good_l4_checksum0
1871                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1872             {
1873               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1874               good_l4_checksum0 =
1875                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1876             }
1877           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1878                              && ! good_l4_checksum1
1879                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1880             {
1881               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1882               good_l4_checksum1 =
1883                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1884             }
1885
1886           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1887
1888           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1889           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1890
1891           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1892           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1893           error0 = (! good_l4_checksum0
1894                     ? IP6_ERROR_UDP_CHECKSUM + type0
1895                     : error0);
1896           error1 = (! good_l4_checksum1
1897                     ? IP6_ERROR_UDP_CHECKSUM + type1
1898                     : error1);
1899
1900           /* Drop packets from unroutable hosts. */
1901           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1902           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1903             {
1904               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1905               error0 = (lm->miss_adj_index == src_adj_index0
1906                         ? IP6_ERROR_SRC_LOOKUP_MISS
1907                         : error0);
1908             }
1909           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL && type1 != IP_BUILTIN_PROTOCOL_ICMP)
1910             {
1911               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1912               error1 = (lm->miss_adj_index == src_adj_index1
1913                         ? IP6_ERROR_SRC_LOOKUP_MISS
1914                         : error1);
1915             }
1916
1917           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1918           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1919
1920           p0->error = error_node->errors[error0];
1921           p1->error = error_node->errors[error1];
1922
1923           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1924                                            to_next, n_left_to_next,
1925                                            pi0, pi1, next0, next1);
1926         }
1927
1928       while (n_left_from > 0 && n_left_to_next > 0)
1929         {
1930           vlib_buffer_t * p0;
1931           ip6_header_t * ip0;
1932           udp_header_t * udp0;
1933           u32 pi0, ip_len0, udp_len0, flags0, next0;
1934           i32 len_diff0;
1935           u8 error0, type0, good_l4_checksum0;
1936
1937           pi0 = to_next[0] = from[0];
1938           from += 1;
1939           n_left_from -= 1;
1940           to_next += 1;
1941           n_left_to_next -= 1;
1942
1943           p0 = vlib_get_buffer (vm, pi0);
1944
1945           ip0 = vlib_buffer_get_current (p0);
1946
1947           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1948           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1949
1950           flags0 = p0->flags;
1951
1952           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1953
1954           udp0 = ip6_next_header (ip0);
1955
1956           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1957           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1958
1959           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1960
1961           /* Verify UDP length. */
1962           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1963           udp_len0 = clib_net_to_host_u16 (udp0->length);
1964
1965           len_diff0 = ip_len0 - udp_len0;
1966
1967           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1968
1969           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1970                              && ! good_l4_checksum0
1971                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1972             {
1973               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1974               good_l4_checksum0 =
1975                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1976             }
1977
1978           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1979
1980           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1981
1982           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1983           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1984           error0 = (! good_l4_checksum0
1985                     ? IP6_ERROR_UDP_CHECKSUM + type0
1986                     : error0);
1987
1988           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1989           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1990             {
1991               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1992               error0 = (lm->miss_adj_index == src_adj_index0
1993                         ? IP6_ERROR_SRC_LOOKUP_MISS
1994                         : error0);
1995             }
1996
1997           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1998
1999           p0->error = error_node->errors[error0];
2000
2001           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2002                                            to_next, n_left_to_next,
2003                                            pi0, next0);
2004         }
2005
2006       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2007     }
2008
2009   return frame->n_vectors;
2010 }
2011
2012 VLIB_REGISTER_NODE (ip6_local_node,static) = {
2013   .function = ip6_local,
2014   .name = "ip6-local",
2015   .vector_size = sizeof (u32),
2016
2017   .format_trace = format_ip6_forward_next_trace,
2018
2019   .n_next_nodes = IP_LOCAL_N_NEXT,
2020   .next_nodes = {
2021     [IP_LOCAL_NEXT_DROP] = "error-drop",
2022     [IP_LOCAL_NEXT_PUNT] = "error-punt",
2023     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
2024     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
2025   },
2026 };
2027
2028 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
2029
2030 void ip6_register_protocol (u32 protocol, u32 node_index)
2031 {
2032   vlib_main_t * vm = vlib_get_main();
2033   ip6_main_t * im = &ip6_main;
2034   ip_lookup_main_t * lm = &im->lookup_main;
2035
2036   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
2037   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
2038 }
2039
2040 typedef enum {
2041   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2042   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
2043   IP6_DISCOVER_NEIGHBOR_N_NEXT,
2044 } ip6_discover_neighbor_next_t;
2045
2046 typedef enum {
2047   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
2048   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
2049   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
2050 } ip6_discover_neighbor_error_t;
2051
2052 static uword
2053 ip6_discover_neighbor (vlib_main_t * vm,
2054                        vlib_node_runtime_t * node,
2055                        vlib_frame_t * frame)
2056 {
2057   vnet_main_t * vnm = vnet_get_main();
2058   ip6_main_t * im = &ip6_main;
2059   ip_lookup_main_t * lm = &im->lookup_main;
2060   u32 * from, * to_next_drop;
2061   uword n_left_from, n_left_to_next_drop;
2062   static f64 time_last_seed_change = -1e100;
2063   static u32 hash_seeds[3];
2064   static uword hash_bitmap[256 / BITS (uword)];
2065   f64 time_now;
2066   int bogus_length;
2067
2068   if (node->flags & VLIB_NODE_FLAG_TRACE)
2069     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2070
2071   time_now = vlib_time_now (vm);
2072   if (time_now - time_last_seed_change > 1e-3)
2073     {
2074       uword i;
2075       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2076                                              sizeof (hash_seeds));
2077       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2078         hash_seeds[i] = r[i];
2079
2080       /* Mark all hash keys as been not-seen before. */
2081       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2082         hash_bitmap[i] = 0;
2083
2084       time_last_seed_change = time_now;
2085     }
2086
2087   from = vlib_frame_vector_args (frame);
2088   n_left_from = frame->n_vectors;
2089
2090   while (n_left_from > 0)
2091     {
2092       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2093                            to_next_drop, n_left_to_next_drop);
2094
2095       while (n_left_from > 0 && n_left_to_next_drop > 0)
2096         {
2097           vlib_buffer_t * p0;
2098           ip6_header_t * ip0;
2099           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2100           uword bm0;
2101           ip_adjacency_t * adj0;
2102           vnet_hw_interface_t * hw_if0;
2103           u32 next0;
2104
2105           pi0 = from[0];
2106
2107           p0 = vlib_get_buffer (vm, pi0);
2108
2109           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2110
2111           ip0 = vlib_buffer_get_current (p0);
2112
2113           adj0 = ip_get_adjacency (lm, adj_index0);
2114
2115           if (adj0->arp.next_hop.ip6.as_u64[0] ||
2116               adj0->arp.next_hop.ip6.as_u64[1]) {
2117             ip0->dst_address.as_u64[0] = adj0->arp.next_hop.ip6.as_u64[0];
2118             ip0->dst_address.as_u64[1] = adj0->arp.next_hop.ip6.as_u64[1];
2119           }
2120
2121           a0 = hash_seeds[0];
2122           b0 = hash_seeds[1];
2123           c0 = hash_seeds[2];
2124
2125           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2126           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2127
2128           a0 ^= sw_if_index0;
2129           b0 ^= ip0->dst_address.as_u32[0];
2130           c0 ^= ip0->dst_address.as_u32[1];
2131
2132           hash_v3_mix32 (a0, b0, c0);
2133
2134           b0 ^= ip0->dst_address.as_u32[2];
2135           c0 ^= ip0->dst_address.as_u32[3];
2136
2137           hash_v3_finalize32 (a0, b0, c0);
2138
2139           c0 &= BITS (hash_bitmap) - 1;
2140           c0 = c0 / BITS (uword);
2141           m0 = (uword) 1 << (c0 % BITS (uword));
2142
2143           bm0 = hash_bitmap[c0];
2144           drop0 = (bm0 & m0) != 0;
2145
2146           /* Mark it as seen. */
2147           hash_bitmap[c0] = bm0 | m0;
2148
2149           from += 1;
2150           n_left_from -= 1;
2151           to_next_drop[0] = pi0;
2152           to_next_drop += 1;
2153           n_left_to_next_drop -= 1;
2154
2155           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2156
2157           /* If the interface is link-down, drop the pkt */
2158           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
2159             drop0 = 1;
2160
2161           p0->error =
2162             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
2163                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
2164           if (drop0)
2165             continue;
2166
2167           {
2168             u32 bi0 = 0;
2169             icmp6_neighbor_solicitation_header_t * h0;
2170             vlib_buffer_t * b0;
2171
2172             h0 = vlib_packet_template_get_packet
2173               (vm, &im->discover_neighbor_packet_template, &bi0);
2174
2175             /*
2176              * Build ethernet header.
2177              * Choose source address based on destination lookup
2178              * adjacency.
2179              */
2180             if (ip6_src_address_for_packet (im, p0, &h0->ip.src_address,
2181                                                 sw_if_index0)) {
2182                 //There is no address on the interface
2183                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
2184                 vlib_buffer_free(vm, &bi0, 1);
2185                 continue;
2186             }
2187
2188             /*
2189              * Destination address is a solicited node multicast address.
2190              * We need to fill in
2191              * the low 24 bits with low 24 bits of target's address.
2192              */
2193             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
2194             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
2195             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
2196
2197             h0->neighbor.target_address = ip0->dst_address;
2198
2199             clib_memcpy (h0->link_layer_option.ethernet_address,
2200                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
2201
2202             /* $$$$ appears we need this; why is the checksum non-zero? */
2203             h0->neighbor.icmp.checksum = 0;
2204             h0->neighbor.icmp.checksum =
2205               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
2206                                                  &bogus_length);
2207
2208             ASSERT (bogus_length == 0);
2209
2210             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2211             b0 = vlib_get_buffer (vm, bi0);
2212             vnet_buffer (b0)->sw_if_index[VLIB_TX]
2213               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
2214
2215             /* Add rewrite/encap string. */
2216             vnet_rewrite_one_header (adj0[0], h0,
2217                                      sizeof (ethernet_header_t));
2218             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2219
2220             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
2221
2222             vlib_set_next_frame_buffer (vm, node, next0, bi0);
2223           }
2224         }
2225
2226       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2227                            n_left_to_next_drop);
2228     }
2229
2230   return frame->n_vectors;
2231 }
2232
2233 static char * ip6_discover_neighbor_error_strings[] = {
2234   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
2235   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
2236   = "neighbor solicitations sent",
2237   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
2238     = "no source address for ND solicitation",
2239 };
2240
2241 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
2242   .function = ip6_discover_neighbor,
2243   .name = "ip6-discover-neighbor",
2244   .vector_size = sizeof (u32),
2245
2246   .format_trace = format_ip6_forward_next_trace,
2247
2248   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
2249   .error_strings = ip6_discover_neighbor_error_strings,
2250
2251   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
2252   .next_nodes = {
2253     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
2254     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
2255   },
2256 };
2257
2258 clib_error_t *
2259 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
2260 {
2261   vnet_main_t * vnm = vnet_get_main();
2262   ip6_main_t * im = &ip6_main;
2263   icmp6_neighbor_solicitation_header_t * h;
2264   ip6_address_t * src;
2265   ip_interface_address_t * ia;
2266   ip_adjacency_t * adj;
2267   vnet_hw_interface_t * hi;
2268   vnet_sw_interface_t * si;
2269   vlib_buffer_t * b;
2270   u32 bi = 0;
2271   int bogus_length;
2272
2273   si = vnet_get_sw_interface (vnm, sw_if_index);
2274
2275   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2276     {
2277       return clib_error_return (0, "%U: interface %U down",
2278                                 format_ip6_address, dst,
2279                                 format_vnet_sw_if_index_name, vnm,
2280                                 sw_if_index);
2281     }
2282
2283   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2284   if (! src)
2285     {
2286       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2287       return clib_error_return
2288         (0, "no matching interface address for destination %U (interface %U)",
2289          format_ip6_address, dst,
2290          format_vnet_sw_if_index_name, vnm, sw_if_index);
2291     }
2292
2293   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
2294
2295   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2296
2297   /* Destination address is a solicited node multicast address.  We need to fill in
2298      the low 24 bits with low 24 bits of target's address. */
2299   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
2300   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
2301   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
2302
2303   h->ip.src_address = src[0];
2304   h->neighbor.target_address = dst[0];
2305
2306   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
2307
2308   h->neighbor.icmp.checksum =
2309     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
2310   ASSERT(bogus_length == 0);
2311
2312   b = vlib_get_buffer (vm, bi);
2313   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2314
2315   /* Add encapsulation string for software interface (e.g. ethernet header). */
2316   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2317   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2318   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2319
2320   {
2321     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2322     u32 * to_next = vlib_frame_vector_args (f);
2323     to_next[0] = bi;
2324     f->n_vectors = 1;
2325     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2326   }
2327
2328   return /* no error */ 0;
2329 }
2330
2331 typedef enum {
2332   IP6_REWRITE_NEXT_DROP,
2333   IP6_REWRITE_NEXT_ICMP_ERROR,
2334 } ip6_rewrite_next_t;
2335
2336 always_inline uword
2337 ip6_rewrite_inline (vlib_main_t * vm,
2338                     vlib_node_runtime_t * node,
2339                     vlib_frame_t * frame,
2340                     int rewrite_for_locally_received_packets)
2341 {
2342   ip_lookup_main_t * lm = &ip6_main.lookup_main;
2343   u32 * from = vlib_frame_vector_args (frame);
2344   u32 n_left_from, n_left_to_next, * to_next, next_index;
2345   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
2346   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2347
2348   n_left_from = frame->n_vectors;
2349   next_index = node->cached_next_index;
2350   u32 cpu_index = os_get_cpu_number();
2351
2352   while (n_left_from > 0)
2353     {
2354       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2355
2356       while (n_left_from >= 4 && n_left_to_next >= 2)
2357         {
2358           ip_adjacency_t * adj0, * adj1;
2359           vlib_buffer_t * p0, * p1;
2360           ip6_header_t * ip0, * ip1;
2361           u32 pi0, rw_len0, next0, error0, adj_index0;
2362           u32 pi1, rw_len1, next1, error1, adj_index1;
2363
2364           /* Prefetch next iteration. */
2365           {
2366             vlib_buffer_t * p2, * p3;
2367
2368             p2 = vlib_get_buffer (vm, from[2]);
2369             p3 = vlib_get_buffer (vm, from[3]);
2370
2371             vlib_prefetch_buffer_header (p2, LOAD);
2372             vlib_prefetch_buffer_header (p3, LOAD);
2373
2374             CLIB_PREFETCH (p2->pre_data, 32, STORE);
2375             CLIB_PREFETCH (p3->pre_data, 32, STORE);
2376
2377             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2378             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2379           }
2380
2381           pi0 = to_next[0] = from[0];
2382           pi1 = to_next[1] = from[1];
2383
2384           from += 2;
2385           n_left_from -= 2;
2386           to_next += 2;
2387           n_left_to_next -= 2;
2388
2389           p0 = vlib_get_buffer (vm, pi0);
2390           p1 = vlib_get_buffer (vm, pi1);
2391
2392           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2393           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2394
2395           /* We should never rewrite a pkt using the MISS adjacency */
2396           ASSERT(adj_index0 && adj_index1);
2397
2398           ip0 = vlib_buffer_get_current (p0);
2399           ip1 = vlib_buffer_get_current (p1);
2400
2401           error0 = error1 = IP6_ERROR_NONE;
2402           next0 = next1 = IP6_REWRITE_NEXT_DROP;
2403
2404           if (! rewrite_for_locally_received_packets)
2405             {
2406               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
2407
2408               /* Input node should have reject packets with hop limit 0. */
2409               ASSERT (ip0->hop_limit > 0);
2410               ASSERT (ip1->hop_limit > 0);
2411
2412               hop_limit0 -= 1;
2413               hop_limit1 -= 1;
2414
2415               ip0->hop_limit = hop_limit0;
2416               ip1->hop_limit = hop_limit1;
2417
2418               /*
2419                * If the hop count drops below 1 when forwarding, generate
2420                * an ICMP response.
2421                */
2422               if (PREDICT_FALSE(hop_limit0 <= 0))
2423                 {
2424                   error0 = IP6_ERROR_TIME_EXPIRED;
2425                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2426                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2427                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2428                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2429                 }
2430               if (PREDICT_FALSE(hop_limit1 <= 0))
2431                 {
2432                   error1 = IP6_ERROR_TIME_EXPIRED;
2433                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
2434                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2435                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
2436                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2437                 }
2438             }
2439
2440           adj0 = ip_get_adjacency (lm, adj_index0);
2441           adj1 = ip_get_adjacency (lm, adj_index1);
2442
2443           if (rewrite_for_locally_received_packets)
2444             {
2445               /*
2446                * If someone sends e.g. an icmp6 w/ src = dst = interface addr,
2447                * we end up here with a local adjacency in hand
2448                */
2449               if (PREDICT_FALSE(adj0->lookup_next_index
2450                                 == IP_LOOKUP_NEXT_LOCAL))
2451                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2452               if (PREDICT_FALSE(adj1->lookup_next_index
2453                                 == IP_LOOKUP_NEXT_LOCAL))
2454                 error1 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2455             }
2456
2457           rw_len0 = adj0[0].rewrite_header.data_bytes;
2458           rw_len1 = adj1[0].rewrite_header.data_bytes;
2459
2460           vlib_increment_combined_counter (&lm->adjacency_counters,
2461                                            cpu_index,
2462                                            adj_index0,
2463                                            /* packet increment */ 0,
2464                                            /* byte increment */ rw_len0);
2465           vlib_increment_combined_counter (&lm->adjacency_counters,
2466                                            cpu_index,
2467                                            adj_index1,
2468                                            /* packet increment */ 0,
2469                                            /* byte increment */ rw_len1);
2470
2471           /* Check MTU of outgoing interface. */
2472           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2473                     ? IP6_ERROR_MTU_EXCEEDED
2474                     : error0);
2475           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2476                     ? IP6_ERROR_MTU_EXCEEDED
2477                     : error1);
2478
2479           /* Don't adjust the buffer for hop count issue; icmp-error node
2480            * wants to see the IP headerr */
2481           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2482             {
2483               p0->current_data -= rw_len0;
2484               p0->current_length += rw_len0;
2485
2486               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2487                   adj0[0].rewrite_header.sw_if_index;
2488               next0 = adj0[0].rewrite_header.next_index;
2489             }
2490           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
2491             {
2492               p1->current_data -= rw_len1;
2493               p1->current_length += rw_len1;
2494
2495               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2496                   adj1[0].rewrite_header.sw_if_index;
2497               next1 = adj1[0].rewrite_header.next_index;
2498             }
2499
2500           /* Guess we are only writing on simple Ethernet header. */
2501           vnet_rewrite_two_headers (adj0[0], adj1[0],
2502                                     ip0, ip1,
2503                                     sizeof (ethernet_header_t));
2504
2505           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2506                                            to_next, n_left_to_next,
2507                                            pi0, pi1, next0, next1);
2508         }
2509
2510       while (n_left_from > 0 && n_left_to_next > 0)
2511         {
2512           ip_adjacency_t * adj0;
2513           vlib_buffer_t * p0;
2514           ip6_header_t * ip0;
2515           u32 pi0, rw_len0;
2516           u32 adj_index0, next0, error0;
2517
2518           pi0 = to_next[0] = from[0];
2519
2520           p0 = vlib_get_buffer (vm, pi0);
2521
2522           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2523
2524           /* We should never rewrite a pkt using the MISS adjacency */
2525           ASSERT(adj_index0);
2526
2527           adj0 = ip_get_adjacency (lm, adj_index0);
2528
2529           ip0 = vlib_buffer_get_current (p0);
2530
2531           error0 = IP6_ERROR_NONE;
2532           next0 = IP6_REWRITE_NEXT_DROP;
2533
2534           /* Check hop limit */
2535           if (! rewrite_for_locally_received_packets)
2536             {
2537               i32 hop_limit0 = ip0->hop_limit;
2538
2539               ASSERT (ip0->hop_limit > 0);
2540
2541               hop_limit0 -= 1;
2542
2543               ip0->hop_limit = hop_limit0;
2544
2545               if (PREDICT_FALSE(hop_limit0 <= 0))
2546                 {
2547                   /*
2548                    * If the hop count drops below 1 when forwarding, generate
2549                    * an ICMP response.
2550                    */
2551                   error0 = IP6_ERROR_TIME_EXPIRED;
2552                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2553                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2554                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2555                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2556                 }
2557             }
2558
2559           if (rewrite_for_locally_received_packets)
2560             {
2561               if (PREDICT_FALSE(adj0->lookup_next_index
2562                                 == IP_LOOKUP_NEXT_LOCAL))
2563                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2564             }
2565
2566           /* Guess we are only writing on simple Ethernet header. */
2567           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2568
2569           /* Update packet buffer attributes/set output interface. */
2570           rw_len0 = adj0[0].rewrite_header.data_bytes;
2571
2572           vlib_increment_combined_counter (&lm->adjacency_counters,
2573                                            cpu_index,
2574                                            adj_index0,
2575                                            /* packet increment */ 0,
2576                                            /* byte increment */ rw_len0);
2577
2578           /* Check MTU of outgoing interface. */
2579           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2580                     ? IP6_ERROR_MTU_EXCEEDED
2581                     : error0);
2582
2583           /* Don't adjust the buffer for hop count issue; icmp-error node
2584            * wants to see the IP headerr */
2585           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2586             {
2587               p0->current_data -= rw_len0;
2588               p0->current_length += rw_len0;
2589
2590               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2591                   adj0[0].rewrite_header.sw_if_index;
2592               next0 = adj0[0].rewrite_header.next_index;
2593             }
2594
2595           p0->error = error_node->errors[error0];
2596
2597           from += 1;
2598           n_left_from -= 1;
2599           to_next += 1;
2600           n_left_to_next -= 1;
2601
2602           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2603                                            to_next, n_left_to_next,
2604                                            pi0, next0);
2605         }
2606
2607       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2608     }
2609
2610   /* Need to do trace after rewrites to pick up new packet data. */
2611   if (node->flags & VLIB_NODE_FLAG_TRACE)
2612     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2613
2614   return frame->n_vectors;
2615 }
2616
2617 static uword
2618 ip6_rewrite_transit (vlib_main_t * vm,
2619                      vlib_node_runtime_t * node,
2620                      vlib_frame_t * frame)
2621 {
2622   return ip6_rewrite_inline (vm, node, frame,
2623                              /* rewrite_for_locally_received_packets */ 0);
2624 }
2625
2626 static uword
2627 ip6_rewrite_local (vlib_main_t * vm,
2628                    vlib_node_runtime_t * node,
2629                    vlib_frame_t * frame)
2630 {
2631   return ip6_rewrite_inline (vm, node, frame,
2632                              /* rewrite_for_locally_received_packets */ 1);
2633 }
2634
2635 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2636   .function = ip6_rewrite_transit,
2637   .name = "ip6-rewrite",
2638   .vector_size = sizeof (u32),
2639
2640   .format_trace = format_ip6_rewrite_trace,
2641
2642   .n_next_nodes = 2,
2643   .next_nodes = {
2644     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2645     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2646   },
2647 };
2648
2649 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit)
2650
2651 VLIB_REGISTER_NODE (ip6_rewrite_local_node) = {
2652   .function = ip6_rewrite_local,
2653   .name = "ip6-rewrite-local",
2654   .vector_size = sizeof (u32),
2655
2656   .sibling_of = "ip6-rewrite",
2657
2658   .format_trace = format_ip6_rewrite_trace,
2659
2660   .n_next_nodes = 0,
2661 };
2662
2663 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local)
2664
2665 /*
2666  * Hop-by-Hop handling
2667  */
2668
2669 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2670
2671 #define foreach_ip6_hop_by_hop_error \
2672 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2673 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2674 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2675
2676 typedef enum {
2677 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2678   foreach_ip6_hop_by_hop_error
2679 #undef _
2680   IP6_HOP_BY_HOP_N_ERROR,
2681 } ip6_hop_by_hop_error_t;
2682
2683 /*
2684  * Primary h-b-h handler trace support
2685  * We work pretty hard on the problem for obvious reasons
2686  */
2687 typedef struct {
2688   u32 next_index;
2689   u32 trace_len;
2690   u8 option_data[256];
2691 } ip6_hop_by_hop_trace_t;
2692
2693 vlib_node_registration_t ip6_hop_by_hop_node;
2694
2695 static char * ip6_hop_by_hop_error_strings[] = {
2696 #define _(sym,string) string,
2697   foreach_ip6_hop_by_hop_error
2698 #undef _
2699 };
2700
2701 static u8 *
2702 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2703 {
2704   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2705   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2706   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2707   ip6_hop_by_hop_header_t *hbh0;
2708   ip6_hop_by_hop_option_t *opt0, *limit0;
2709   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2710
2711   u8 type0;
2712
2713   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2714
2715   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2716               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2717
2718   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2719   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2720
2721   while (opt0 < limit0) {
2722     type0 = opt0->type;
2723     switch (type0) {
2724     case 0: /* Pad, just stop */
2725       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2726       break;
2727
2728     default:
2729       if (hm->trace[type0]) {
2730         s = (*hm->trace[type0])(s, opt0);
2731       } else {
2732         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2733       }
2734       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2735       break;
2736     }
2737   }
2738   return s;
2739 }
2740
2741 /*
2742  * Process the Hop-by-Hop Options header
2743  */
2744 static uword
2745 ip6_hop_by_hop (vlib_main_t * vm,
2746                 vlib_node_runtime_t * node,
2747                 vlib_frame_t * frame)
2748 {
2749   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2750   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2751   u32 n_left_from, *from, *to_next;
2752   ip_lookup_next_t next_index;
2753   ip6_main_t * im = &ip6_main;
2754   ip_lookup_main_t *lm = &im->lookup_main;
2755
2756   from = vlib_frame_vector_args (frame);
2757   n_left_from = frame->n_vectors;
2758   next_index = node->cached_next_index;
2759
2760   while (n_left_from > 0) {
2761     u32 n_left_to_next;
2762
2763     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2764
2765     while (n_left_from > 0 && n_left_to_next > 0) {
2766       u32 bi0;
2767       vlib_buffer_t * b0;
2768       u32 next0;
2769       ip6_header_t * ip0;
2770       ip6_hop_by_hop_header_t *hbh0;
2771       ip6_hop_by_hop_option_t *opt0, *limit0;
2772       u8 type0;
2773       u8 error0 = 0;
2774
2775       /* Speculatively enqueue b0 to the current next frame */
2776       bi0 = from[0];
2777       to_next[0] = bi0;
2778       from += 1;
2779       to_next += 1;
2780       n_left_from -= 1;
2781       n_left_to_next -= 1;
2782
2783       b0 = vlib_get_buffer (vm, bi0);
2784       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2785       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2786       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2787       next0 = adj0->lookup_next_index;
2788
2789       ip0 = vlib_buffer_get_current (b0);
2790       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2791       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2792       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2793
2794       /*
2795        * Basic validity checks
2796        */
2797       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2798         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2799         next0 = IP_LOOKUP_NEXT_DROP;
2800         goto out0;
2801       }
2802
2803       /* Scan the set of h-b-h options, process ones that we understand */
2804       while (opt0 < limit0) {
2805         type0 = opt0->type;
2806         switch (type0) {
2807         case 0: /* Pad1 */
2808           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2809           continue;
2810         case 1: /* PadN */
2811           break;
2812         default:
2813           if (hm->options[type0]) {
2814             if ((*hm->options[type0])(b0, ip0, opt0) < 0) {
2815               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2816               goto out0;
2817             }
2818           } else {
2819             /* Unrecognized mandatory option, check the two high order bits */
2820             switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS) {
2821             case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2822               break;
2823             case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2824               next0 = IP_LOOKUP_NEXT_DROP;
2825               break;
2826             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2827               next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2828               icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2829                                           ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2830               break;
2831             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2832               if (!ip6_address_is_multicast(&ip0->dst_address)) {
2833                 next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2834                 icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2835                                             ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2836               } else {
2837                 next0 =  IP_LOOKUP_NEXT_DROP;
2838               }
2839               break;
2840             }
2841             error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2842             goto out0;
2843           }
2844         }
2845         opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2846       }
2847
2848     out0:
2849       /* Has the classifier flagged this buffer for special treatment? */
2850       if ((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index == OI_DECAP))
2851         next0 = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2852
2853       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2854         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2855         u32 trace_len = (hbh0->length + 1) << 3;
2856         t->next_index = next0;
2857         /* Capture the h-b-h option verbatim */
2858         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2859         t->trace_len = trace_len;
2860         clib_memcpy(t->option_data, hbh0, trace_len);
2861       }
2862
2863       b0->error = error_node->errors[error0];
2864
2865       /* verify speculative enqueue, maybe switch current next frame */
2866       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2867     }
2868     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2869   }
2870   return frame->n_vectors;
2871 }
2872
2873 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2874   .function = ip6_hop_by_hop,
2875   .name = "ip6-hop-by-hop",
2876   .sibling_of = "ip6-lookup",
2877   .vector_size = sizeof (u32),
2878   .format_trace = format_ip6_hop_by_hop_trace,
2879   .type = VLIB_NODE_TYPE_INTERNAL,
2880   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2881   .error_strings = ip6_hop_by_hop_error_strings,
2882   .n_next_nodes = 0,
2883 };
2884
2885 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop)
2886
2887 static clib_error_t *
2888 ip6_hop_by_hop_init (vlib_main_t * vm)
2889 {
2890   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2891   memset(hm->options, 0, sizeof(hm->options));
2892   memset(hm->trace, 0, sizeof(hm->trace));
2893
2894   return (0);
2895 }
2896
2897 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2898
2899 int
2900 ip6_hbh_register_option (u8 option,
2901                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2902                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2903 {
2904   ip6_main_t * im = &ip6_main;
2905   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2906
2907   ASSERT (option < ARRAY_LEN (hm->options));
2908
2909   /* Already registered */
2910   if (hm->options[option])
2911     return (-1);
2912
2913   hm->options[option] = options;
2914   hm->trace[option] = trace;
2915
2916   /* Set global variable */
2917   im->hbh_enabled = 1;
2918
2919   return (0);
2920 }
2921
2922 int
2923 ip6_hbh_unregister_option (u8 option)
2924 {
2925   ip6_main_t * im = &ip6_main;
2926   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2927
2928   ASSERT (option < ARRAY_LEN (hm->options));
2929
2930   /* Not registered */
2931   if (!hm->options[option])
2932     return (-1);
2933
2934   hm->options[option] = NULL;
2935   hm->trace[option] = NULL;
2936
2937   /* Disable global knob if this was the last option configured */
2938   int i;
2939   bool found = false;
2940   for (i = 0; i < 256; i++) {
2941     if (hm->options[option]) {
2942       found = true;
2943       break;
2944     }
2945   }
2946   if (!found)
2947     im->hbh_enabled = 0;
2948
2949   return (0);
2950 }
2951
2952 /* Global IP6 main. */
2953 ip6_main_t ip6_main;
2954
2955 static clib_error_t *
2956 ip6_lookup_init (vlib_main_t * vm)
2957 {
2958   ip6_main_t * im = &ip6_main;
2959   clib_error_t * error;
2960   uword i;
2961
2962   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2963     {
2964       u32 j, i0, i1;
2965
2966       i0 = i / 32;
2967       i1 = i % 32;
2968
2969       for (j = 0; j < i0; j++)
2970         im->fib_masks[i].as_u32[j] = ~0;
2971
2972       if (i1)
2973         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2974     }
2975
2976   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2977
2978   if (im->lookup_table_nbuckets == 0)
2979     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2980
2981   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2982
2983   if (im->lookup_table_size == 0)
2984     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2985
2986   BV(clib_bihash_init) (&im->ip6_lookup_table, "ip6 lookup table",
2987                         im->lookup_table_nbuckets,
2988                         im->lookup_table_size);
2989
2990   /* Create FIB with index 0 and table id of 0. */
2991   find_ip6_fib_by_table_index_or_id (im, /* table id */ 0, IP6_ROUTE_FLAG_TABLE_ID);
2992
2993   {
2994     pg_node_t * pn;
2995     pn = pg_get_node (ip6_lookup_node.index);
2996     pn->unformat_edit = unformat_pg_ip6_header;
2997   }
2998
2999   /* Unless explicitly configured, don't process HBH options */
3000   im->hbh_enabled = 0;
3001
3002   {
3003     icmp6_neighbor_solicitation_header_t p;
3004
3005     memset (&p, 0, sizeof (p));
3006
3007     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
3008     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
3009                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
3010     p.ip.protocol = IP_PROTOCOL_ICMP6;
3011     p.ip.hop_limit = 255;
3012     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
3013
3014     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
3015
3016     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
3017     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
3018
3019     vlib_packet_template_init (vm,
3020                                &im->discover_neighbor_packet_template,
3021                                &p, sizeof (p),
3022                                /* alloc chunk size */ 8,
3023                                "ip6 neighbor discovery");
3024   }
3025
3026   error = ip6_feature_init (vm, im);
3027
3028   return error;
3029 }
3030
3031 VLIB_INIT_FUNCTION (ip6_lookup_init);
3032
3033 static clib_error_t *
3034 add_del_ip6_interface_table (vlib_main_t * vm,
3035                              unformat_input_t * input,
3036                              vlib_cli_command_t * cmd)
3037 {
3038   vnet_main_t * vnm = vnet_get_main();
3039   clib_error_t * error = 0;
3040   u32 sw_if_index, table_id;
3041
3042   sw_if_index = ~0;
3043
3044   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
3045     {
3046       error = clib_error_return (0, "unknown interface `%U'",
3047                                  format_unformat_error, input);
3048       goto done;
3049     }
3050
3051   if (unformat (input, "%d", &table_id))
3052     ;
3053   else
3054     {
3055       error = clib_error_return (0, "expected table id `%U'",
3056                                  format_unformat_error, input);
3057       goto done;
3058     }
3059
3060   {
3061     ip6_main_t * im = &ip6_main;
3062     ip6_fib_t * fib =
3063       find_ip6_fib_by_table_index_or_id (im, table_id, IP6_ROUTE_FLAG_TABLE_ID);
3064
3065     if (fib)
3066       {
3067         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
3068         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
3069     }
3070   }
3071
3072  done:
3073   return error;
3074 }
3075
3076 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
3077   .path = "set interface ip6 table",
3078   .function = add_del_ip6_interface_table,
3079   .short_help = "set interface ip6 table <intfc> <table-id>"
3080 };
3081
3082 void
3083 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
3084                                                   u8 *mac)
3085 {
3086   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
3087   /* Invert the "u" bit */
3088   ip->as_u8 [8] = mac[0] ^ (1<<1);
3089   ip->as_u8 [9] = mac[1];
3090   ip->as_u8 [10] = mac[2];
3091   ip->as_u8 [11] = 0xFF;
3092   ip->as_u8 [12] = 0xFE;
3093   ip->as_u8 [13] = mac[3];
3094   ip->as_u8 [14] = mac[4];
3095   ip->as_u8 [15] = mac[5];
3096 }
3097
3098 void
3099 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
3100                                                   ip6_address_t *ip)
3101 {
3102   /* Invert the previously inverted "u" bit */
3103   mac[0] = ip->as_u8 [8] ^ (1<<1);
3104   mac[1] = ip->as_u8 [9];
3105   mac[2] = ip->as_u8 [10];
3106   mac[3] = ip->as_u8 [13];
3107   mac[4] = ip->as_u8 [14];
3108   mac[5] = ip->as_u8 [15];
3109 }
3110
3111 static clib_error_t *
3112 test_ip6_link_command_fn (vlib_main_t * vm,
3113                           unformat_input_t * input,
3114                           vlib_cli_command_t * cmd)
3115 {
3116   u8 mac[6];
3117   ip6_address_t _a, *a = &_a;
3118
3119   if (unformat (input, "%U", unformat_ethernet_address, mac))
3120     {
3121       ip6_link_local_address_from_ethernet_mac_address (a, mac);
3122       vlib_cli_output (vm, "Link local address: %U",
3123                        format_ip6_address, a);
3124       ip6_ethernet_mac_address_from_link_local_address (mac, a);
3125       vlib_cli_output (vm, "Original MAC address: %U",
3126                        format_ethernet_address, mac);
3127     }
3128
3129   return 0;
3130 }
3131
3132 VLIB_CLI_COMMAND (test_link_command, static) = {
3133   .path = "test ip6 link",
3134   .function = test_ip6_link_command_fn,
3135   .short_help = "test ip6 link <mac-address>",
3136 };
3137
3138 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
3139 {
3140   ip6_main_t * im6 = &ip6_main;
3141   ip6_fib_t * fib;
3142   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
3143
3144   if (p == 0)
3145     return -1;
3146
3147   fib = vec_elt_at_index (im6->fibs, p[0]);
3148
3149   fib->flow_hash_config = flow_hash_config;
3150   return 1;
3151 }
3152
3153 static clib_error_t *
3154 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
3155                               unformat_input_t * input,
3156                               vlib_cli_command_t * cmd)
3157 {
3158   int matched = 0;
3159   u32 table_id = 0;
3160   u32 flow_hash_config = 0;
3161   int rv;
3162
3163   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3164     if (unformat (input, "table %d", &table_id))
3165       matched = 1;
3166 #define _(a,v) \
3167     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3168     foreach_flow_hash_bit
3169 #undef _
3170     else break;
3171   }
3172
3173   if (matched == 0)
3174     return clib_error_return (0, "unknown input `%U'",
3175                               format_unformat_error, input);
3176
3177   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
3178   switch (rv)
3179     {
3180     case 1:
3181       break;
3182
3183     case -1:
3184       return clib_error_return (0, "no such FIB table %d", table_id);
3185
3186     default:
3187       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3188       break;
3189     }
3190
3191   return 0;
3192 }
3193
3194 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
3195     .path = "set ip6 flow-hash",
3196     .short_help =
3197     "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3198     .function = set_ip6_flow_hash_command_fn,
3199 };
3200
3201 static clib_error_t *
3202 show_ip6_local_command_fn (vlib_main_t * vm,
3203                            unformat_input_t * input,
3204                            vlib_cli_command_t * cmd)
3205 {
3206   ip6_main_t * im = &ip6_main;
3207   ip_lookup_main_t * lm = &im->lookup_main;
3208   int i;
3209
3210   vlib_cli_output (vm, "Protocols handled by ip6_local");
3211   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
3212     {
3213       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3214         vlib_cli_output (vm, "%d", i);
3215     }
3216   return 0;
3217 }
3218
3219
3220
3221 VLIB_CLI_COMMAND (show_ip_local, static) = {
3222   .path = "show ip6 local",
3223   .function = show_ip6_local_command_fn,
3224   .short_help = "Show ip6 local protocol table",
3225 };
3226
3227 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3228                                  u32 table_index)
3229 {
3230   vnet_main_t * vnm = vnet_get_main();
3231   vnet_interface_main_t * im = &vnm->interface_main;
3232   ip6_main_t * ipm = &ip6_main;
3233   ip_lookup_main_t * lm = &ipm->lookup_main;
3234   vnet_classify_main_t * cm = &vnet_classify_main;
3235
3236   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3237     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3238
3239   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3240     return VNET_API_ERROR_NO_SUCH_ENTRY;
3241
3242   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3243   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3244
3245   return 0;
3246 }
3247
3248 static clib_error_t *
3249 set_ip6_classify_command_fn (vlib_main_t * vm,
3250                              unformat_input_t * input,
3251                              vlib_cli_command_t * cmd)
3252 {
3253   u32 table_index = ~0;
3254   int table_index_set = 0;
3255   u32 sw_if_index = ~0;
3256   int rv;
3257
3258   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3259     if (unformat (input, "table-index %d", &table_index))
3260       table_index_set = 1;
3261     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3262                        vnet_get_main(), &sw_if_index))
3263         ;
3264     else
3265         break;
3266   }
3267
3268   if (table_index_set == 0)
3269       return clib_error_return (0, "classify table-index must be specified");
3270
3271   if (sw_if_index == ~0)
3272     return clib_error_return (0, "interface / subif must be specified");
3273
3274   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3275
3276   switch (rv)
3277     {
3278     case 0:
3279       break;
3280
3281     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3282       return clib_error_return (0, "No such interface");
3283
3284     case VNET_API_ERROR_NO_SUCH_ENTRY:
3285       return clib_error_return (0, "No such classifier table");
3286     }
3287   return 0;
3288 }
3289
3290 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3291     .path = "set ip6 classify",
3292     .short_help =
3293     "set ip6 classify intfc <int> table-index <index>",
3294     .function = set_ip6_classify_command_fn,
3295 };
3296
3297 static clib_error_t *
3298 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3299 {
3300   ip6_main_t * im = &ip6_main;
3301   uword heapsize = 0;
3302   u32 tmp;
3303   u32 nbuckets = 0;
3304
3305   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3306     if (unformat (input, "hash-buckets %d", &tmp))
3307       nbuckets = tmp;
3308     else if (unformat (input, "heap-size %dm", &tmp))
3309       heapsize = ((u64)tmp) << 20;
3310     else if (unformat (input, "heap-size %dM", &tmp))
3311       heapsize = ((u64)tmp) << 20;
3312     else if (unformat (input, "heap-size %dg", &tmp))
3313       heapsize = ((u64)tmp) << 30;
3314     else if (unformat (input, "heap-size %dG", &tmp))
3315       heapsize = ((u64)tmp) << 30;
3316     else
3317       return clib_error_return (0, "unknown input '%U'",
3318                                 format_unformat_error, input);
3319   }
3320
3321   im->lookup_table_nbuckets = nbuckets;
3322   im->lookup_table_size = heapsize;
3323
3324   return 0;
3325 }
3326
3327 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3328