vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45
  46 #include <vppinfra/bihash_template.c>
  47
  48 static void compute_prefix_lengths_in_search_order (ip6_main_t * im)
  49 {
  50   int i;
  51   vec_reset_length (im->prefix_lengths_in_search_order);
  52   /* Note: bitmap reversed so this is in fact a longest prefix match */
  53   clib_bitmap_foreach (i, im->non_empty_dst_address_length_bitmap,
  54   ({
  55     int dst_address_length = 128 - i;
  56     vec_add1 (im->prefix_lengths_in_search_order, dst_address_length);
  57   }));
  58 }
  59
  60 u32
  61 ip6_fib_lookup_with_table (ip6_main_t * im, u32 fib_index, ip6_address_t * dst)
  62 {
  63   ip_lookup_main_t * lm = &im->lookup_main;
  64   int i, len;
  65   int rv;
  66   BVT(clib_bihash_kv) kv, value;
  67   u64 fib;
  68
  69   len = vec_len (im->prefix_lengths_in_search_order);
  70
  71   kv.key[0] = dst->as_u64[0];
  72   kv.key[1] = dst->as_u64[1];
  73   fib = ((u64)((fib_index))<<32);
  74
  75   for (i = 0; i < len; i++)
  76     {
  77       int dst_address_length = im->prefix_lengths_in_search_order[i];
  78       ip6_address_t * mask = &im->fib_masks[dst_address_length];
  79
  80       ASSERT(dst_address_length >= 0 && dst_address_length <= 128);
  81       //As lengths are decreasing, masks are increasingly specific.
  82       kv.key[0] &= mask->as_u64[0];
  83       kv.key[1] &= mask->as_u64[1];
  84       kv.key[2] = fib | dst_address_length;
  85
  86       rv = BV(clib_bihash_search_inline_2)(&im->ip6_lookup_table, &kv, &value);
  87       if (rv == 0)
  88         return value.value;
  89     }
  90
  91   return lm->miss_adj_index;
  92 }
  93
  94 u32 ip6_fib_lookup (ip6_main_t * im, u32 sw_if_index, ip6_address_t * dst)
  95 {
  96     u32 fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
  97     return ip6_fib_lookup_with_table (im, fib_index, dst);
  98 }
  99
 100 void
 101 vnet_ip6_fib_init (ip6_main_t * im, u32 fib_index)
 102 {
 103   ip_lookup_main_t * lm = &im->lookup_main;
 104   ip6_add_del_route_args_t a;
 105   ip_adjacency_t * adj;
 106
 107   memset(&a, 0x0, sizeof(ip6_add_del_route_args_t));
 108
 109   a.table_index_or_table_id = fib_index;
 110   a.flags = (IP6_ROUTE_FLAG_ADD
 111              | IP6_ROUTE_FLAG_FIB_INDEX
 112              | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 113              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
 114
 115   /* Add ff02::1:ff00:0/104 via local route for all tables.
 116      This is required for neighbor discovery to work. */
 117   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 118                           &a.adj_index);
 119   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 120   adj->if_address_index = ~0;
 121   adj->rewrite_header.data_bytes = 0;
 122
 123   ip6_set_solicited_node_multicast_address (&a.dst_address, 0);
 124
 125   a.dst_address_length = 104;
 126   ip6_add_del_route (im, &a);
 127
 128   /* Add all-routers multicast address via local route for all tables */
 129   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 130                           &a.adj_index);
 131   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 132   adj->if_address_index = ~0;
 133   adj->rewrite_header.data_bytes = 0;
 134
 135   ip6_set_reserved_multicast_address (&a.dst_address,
 136                                       IP6_MULTICAST_SCOPE_link_local,
 137                                       IP6_MULTICAST_GROUP_ID_all_routers);
 138
 139   a.dst_address_length = 128;
 140   ip6_add_del_route (im, &a);
 141
 142   /* Add all-nodes multicast address via local route for all tables */
 143   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 144                           &a.adj_index);
 145   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 146   adj->if_address_index = ~0;
 147   adj->rewrite_header.data_bytes = 0;
 148
 149   ip6_set_reserved_multicast_address (&a.dst_address,
 150                                       IP6_MULTICAST_SCOPE_link_local,
 151                                       IP6_MULTICAST_GROUP_ID_all_hosts);
 152
 153   a.dst_address_length = 128;
 154   ip6_add_del_route (im, &a);
 155
 156   /* Add all-mldv2  multicast address via local route for all tables */
 157   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 158                           &a.adj_index);
 159   adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
 160   adj->if_address_index = ~0;
 161   adj->rewrite_header.data_bytes = 0;
 162
 163   ip6_set_reserved_multicast_address (&a.dst_address,
 164                                       IP6_MULTICAST_SCOPE_link_local,
 165                                       IP6_MULTICAST_GROUP_ID_mldv2_routers);
 166
 167   a.dst_address_length = 128;
 168   ip6_add_del_route (im, &a);
 169 }
 170
 171 static ip6_fib_t *
 172 create_fib_with_table_id (ip6_main_t * im, u32 table_id)
 173 {
 174   ip6_fib_t * fib;
 175   hash_set (im->fib_index_by_table_id, table_id, vec_len (im->fibs));
 176   vec_add2 (im->fibs, fib, 1);
 177   fib->table_id = table_id;
 178   fib->index = fib - im->fibs;
 179   fib->flow_hash_config = IP_FLOW_HASH_DEFAULT;
 180   vnet_ip6_fib_init (im, fib->index);
 181   return fib;
 182 }
 183
 184 ip6_fib_t *
 185 find_ip6_fib_by_table_index_or_id (ip6_main_t * im, u32 table_index_or_id, u32 flags)
 186 {
 187   uword * p, fib_index;
 188
 189   fib_index = table_index_or_id;
 190   if (! (flags & IP6_ROUTE_FLAG_FIB_INDEX))
 191     {
 192       if (table_index_or_id == ~0) {
 193         table_index_or_id = 0;
 194         while (hash_get (im->fib_index_by_table_id, table_index_or_id)) {
 195           table_index_or_id++;
 196         }
 197         return create_fib_with_table_id (im, table_index_or_id);
 198       }
 199
 200       p = hash_get (im->fib_index_by_table_id, table_index_or_id);
 201       if (! p)
 202         return create_fib_with_table_id (im, table_index_or_id);
 203       fib_index = p[0];
 204     }
 205   return vec_elt_at_index (im->fibs, fib_index);
 206 }
 207
 208 void ip6_add_del_route (ip6_main_t * im, ip6_add_del_route_args_t * a)
 209 {
 210   ip_lookup_main_t * lm = &im->lookup_main;
 211   ip6_fib_t * fib;
 212   ip6_address_t dst_address;
 213   u32 dst_address_length, adj_index;
 214   uword is_del;
 215   u32 old_adj_index = ~0;
 216   BVT(clib_bihash_kv) kv, value;
 217
 218   vlib_smp_unsafe_warning();
 219
 220   is_del = (a->flags & IP6_ROUTE_FLAG_DEL) != 0;
 221
 222   /* Either create new adjacency or use given one depending on arguments. */
 223   if (a->n_add_adj > 0)
 224     {
 225       ip_add_adjacency (lm, a->add_adj, a->n_add_adj, &adj_index);
 226       ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
 227     }
 228   else
 229     adj_index = a->adj_index;
 230
 231   dst_address = a->dst_address;
 232   dst_address_length = a->dst_address_length;
 233   fib = find_ip6_fib_by_table_index_or_id (im, a->table_index_or_table_id,
 234                                            a->flags);
 235
 236   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 237   ip6_address_mask (&dst_address, &im->fib_masks[dst_address_length]);
 238
 239   /* refcount accounting */
 240   if (is_del)
 241     {
 242       ASSERT (im->dst_address_length_refcounts[dst_address_length] > 0);
 243       if (--im->dst_address_length_refcounts[dst_address_length] == 0)
 244         {
 245           im->non_empty_dst_address_length_bitmap =
 246             clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 247                              128 - dst_address_length, 0);
 248           compute_prefix_lengths_in_search_order (im);
 249         }
 250     }
 251   else
 252     {
 253       im->dst_address_length_refcounts[dst_address_length]++;
 254
 255       im->non_empty_dst_address_length_bitmap =
 256         clib_bitmap_set (im->non_empty_dst_address_length_bitmap,
 257                              128 - dst_address_length, 1);
 258       compute_prefix_lengths_in_search_order (im);
 259     }
 260
 261   kv.key[0] = dst_address.as_u64[0];
 262   kv.key[1] = dst_address.as_u64[1];
 263   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 264
 265   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 266     old_adj_index = value.value;
 267
 268   if (is_del)
 269     BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 0 /* is_add */);
 270   else
 271     {
 272       /* Make sure adj index is valid. */
 273       if (CLIB_DEBUG > 0)
 274         (void) ip_get_adjacency (lm, adj_index);
 275
 276       kv.value = adj_index;
 277
 278       BV(clib_bihash_add_del) (&im->ip6_lookup_table, &kv, 1 /* is_add */);
 279     }
 280
 281   /* Avoid spurious reference count increments */
 282   if (old_adj_index == adj_index
 283       && adj_index != ~0
 284       && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
 285     {
 286       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
 287       if (adj->share_count > 0)
 288         adj->share_count --;
 289     }
 290
 291   /* Delete old adjacency index if present and changed. */
 292   {
 293     if (! (a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY)
 294         && old_adj_index != ~0
 295         && old_adj_index != adj_index)
 296       ip_del_adjacency (lm, old_adj_index);
 297   }
 298 }
 299
 300 void
 301 ip6_add_del_route_next_hop (ip6_main_t * im,
 302                             u32 flags,
 303                             ip6_address_t * dst_address,
 304                             u32 dst_address_length,
 305                             ip6_address_t * next_hop,
 306                             u32 next_hop_sw_if_index,
 307                             u32 next_hop_weight, u32 adj_index,
 308                             u32 explicit_fib_index)
 309 {
 310   vnet_main_t * vnm = vnet_get_main();
 311   ip_lookup_main_t * lm = &im->lookup_main;
 312   u32 fib_index;
 313   ip6_fib_t * fib;
 314   ip6_address_t masked_dst_address;
 315   u32 old_mp_adj_index, new_mp_adj_index;
 316   u32 dst_adj_index, nh_adj_index;
 317   int rv;
 318   ip_adjacency_t * dst_adj;
 319   ip_multipath_adjacency_t * old_mp, * new_mp;
 320   int is_del = (flags & IP6_ROUTE_FLAG_DEL) != 0;
 321   int is_interface_next_hop;
 322   clib_error_t * error = 0;
 323   uword * nh_result;
 324   BVT(clib_bihash_kv) kv, value;
 325
 326   vlib_smp_unsafe_warning();
 327
 328   if (explicit_fib_index == (u32)~0)
 329     fib_index = vec_elt (im->fib_index_by_sw_if_index, next_hop_sw_if_index);
 330   else
 331     fib_index = explicit_fib_index;
 332
 333   fib = vec_elt_at_index (im->fibs, fib_index);
 334
 335   /* Lookup next hop to be added or deleted. */
 336   is_interface_next_hop = ip6_address_is_zero (next_hop);
 337   if (adj_index == (u32)~0)
 338     {
 339       if (is_interface_next_hop)
 340         {
 341           nh_result = hash_get (im->interface_route_adj_index_by_sw_if_index,
 342                                 next_hop_sw_if_index);
 343           if (nh_result)
 344             nh_adj_index = *nh_result;
 345           else
 346             {
 347               ip_adjacency_t * adj;
 348               adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
 349                                       &nh_adj_index);
 350               ip6_adjacency_set_interface_route (vnm, adj,
 351                                                  next_hop_sw_if_index, ~0);
 352               ip_call_add_del_adjacency_callbacks
 353                 (lm, nh_adj_index, /* is_del */ 0);
 354               hash_set (im->interface_route_adj_index_by_sw_if_index,
 355                         next_hop_sw_if_index, nh_adj_index);
 356             }
 357         }
 358       else
 359         {
 360           /* Look for the interface /128 route */
 361           kv.key[0] = next_hop->as_u64[0];
 362           kv.key[1] = next_hop->as_u64[1];
 363           kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 364
 365           if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 366           {
 367             ip_adjacency_t * adj;
 368             nh_adj_index = ip6_fib_lookup_with_table (im, fib_index, next_hop);
 369             adj = ip_get_adjacency (lm, nh_adj_index);
 370             /* if ND interface adjacencty is present, we need to
 371                              install ND adjaceny for specific next hop */
 372             if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
 373                 adj->arp.next_hop.ip6.as_u64[0] == 0 &&
 374                 adj->arp.next_hop.ip6.as_u64[1] == 0)
 375             {
 376               nh_adj_index = vnet_ip6_neighbor_glean_add(fib_index, next_hop);
 377             }
 378             else
 379             {
 380               ip_adjacency_t add_adj;
 381               memset (&add_adj, 0, sizeof(add_adj));
 382               add_adj.n_adj = 1;
 383               add_adj.lookup_next_index = IP_LOOKUP_NEXT_INDIRECT;
 384               add_adj.indirect.next_hop.ip6.as_u64[0] = next_hop->as_u64[0];
 385               add_adj.indirect.next_hop.ip6.as_u64[1] = next_hop->as_u64[1];
 386               add_adj.explicit_fib_index = explicit_fib_index;
 387               ip_add_adjacency (lm, &add_adj, 1, &nh_adj_index);
 388             }
 389           }
 390           else
 391             nh_adj_index = value.value;
 392
 393         }
 394     }
 395   else
 396     {
 397       /* Look for the interface /128 route */
 398       kv.key[0] = next_hop->as_u64[0];
 399       kv.key[1] = next_hop->as_u64[1];
 400       kv.key[2] = ((u64)((fib - im->fibs))<<32) | 128;
 401
 402       if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) < 0)
 403         {
 404           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 405           error = clib_error_return (0, "next-hop %U/128 not in FIB",
 406                                      format_ip6_address, next_hop);
 407           goto done;
 408         }
 409
 410       nh_adj_index = value.value;
 411     }
 412
 413   ASSERT (dst_address_length < ARRAY_LEN (im->fib_masks));
 414   masked_dst_address = dst_address[0];
 415   ip6_address_mask (&masked_dst_address, &im->fib_masks[dst_address_length]);
 416
 417   kv.key[0] = masked_dst_address.as_u64[0];
 418   kv.key[1] = masked_dst_address.as_u64[1];
 419   kv.key[2] = ((u64)((fib - im->fibs))<<32) | dst_address_length;
 420
 421   rv = BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value);
 422
 423   if (rv == 0)
 424     {
 425       dst_adj_index = value.value;
 426       dst_adj = ip_get_adjacency (lm, dst_adj_index);
 427     }
 428   else
 429     {
 430       /* For deletes destination must be known. */
 431       if (is_del)
 432         {
 433           vnm->api_errno = VNET_API_ERROR_UNKNOWN_DESTINATION;
 434           error = clib_error_return (0, "unknown destination %U/%d",
 435                                      format_ip6_address, dst_address,
 436                                      dst_address_length);
 437           goto done;
 438         }
 439
 440       dst_adj_index = ~0;
 441       dst_adj = 0;
 442     }
 443
 444   /* Ignore adds of X/128 with next hop of X. */
 445   if (! is_del
 446       && dst_address_length == 128
 447       && ip6_address_is_equal (dst_address, next_hop))
 448     {
 449       vnm->api_errno = VNET_API_ERROR_PREFIX_MATCHES_NEXT_HOP;
 450       error = clib_error_return (0, "prefix matches next hop %U/%d",
 451                                  format_ip6_address, dst_address,
 452                                  dst_address_length);
 453       goto done;
 454     }
 455
 456   /* Destination is not known and default weight is set so add route
 457      to existing non-multipath adjacency */
 458   if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
 459   {
 460     /* create / delete additional mapping of existing adjacency */
 461     ip6_add_del_route_args_t a;
 462     ip_adjacency_t * nh_adj = ip_get_adjacency (lm, nh_adj_index);
 463
 464     a.table_index_or_table_id = fib_index;
 465     a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 466         | IP6_ROUTE_FLAG_FIB_INDEX
 467         | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 468         | (flags & (IP6_ROUTE_FLAG_NO_REDISTRIBUTE
 469             | IP6_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
 470     a.dst_address = dst_address[0];
 471     a.dst_address_length = dst_address_length;
 472     a.adj_index = nh_adj_index;
 473     a.add_adj = 0;
 474     a.n_add_adj = 0;
 475
 476     ip6_add_del_route (im, &a);
 477     /* adjust share count. This cannot be the only use of the adjacency */
 478     nh_adj->share_count += is_del ? -1 : 1;
 479
 480     goto done;
 481   }
 482
 483   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 484
 485   if (! ip_multipath_adjacency_add_del_next_hop
 486       (lm, is_del,
 487        dst_adj ? dst_adj->heap_handle : ~0,
 488        nh_adj_index,
 489        next_hop_weight,
 490        &new_mp_adj_index))
 491     {
 492       vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_FOUND_MP;
 493       error = clib_error_return
 494         (0, "requested deleting next-hop %U not found in multi-path",
 495          format_ip6_address, next_hop);
 496       goto done;
 497     }
 498
 499   old_mp = new_mp = 0;
 500   if (old_mp_adj_index != ~0)
 501     old_mp = vec_elt_at_index (lm->multipath_adjacencies, old_mp_adj_index);
 502   if (new_mp_adj_index != ~0)
 503     new_mp = vec_elt_at_index (lm->multipath_adjacencies, new_mp_adj_index);
 504
 505   if (old_mp != new_mp)
 506     {
 507       ip6_add_del_route_args_t a;
 508       ip_adjacency_t * adj;
 509
 510       a.table_index_or_table_id = fib_index;
 511       a.flags = ((is_del ? IP6_ROUTE_FLAG_DEL : IP6_ROUTE_FLAG_ADD)
 512                  | IP6_ROUTE_FLAG_FIB_INDEX
 513                  | IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY
 514                  | (flags & IP6_ROUTE_FLAG_NO_REDISTRIBUTE));
 515       a.dst_address = dst_address[0];
 516       a.dst_address_length = dst_address_length;
 517       a.adj_index = new_mp ? new_mp->adj_index : dst_adj_index;
 518       a.add_adj = 0;
 519       a.n_add_adj = 0;
 520
 521       ip6_add_del_route (im, &a);
 522
 523       adj = ip_get_adjacency (lm, new_mp ? new_mp->adj_index : dst_adj_index);
 524       if (adj->n_adj == 1)
 525         adj->share_count += is_del ? -1 : 1;
 526     }
 527
 528  done:
 529   if (error)
 530     clib_error_report (error);
 531 }
 532
 533 u32
 534 ip6_get_route (ip6_main_t * im,
 535                u32 table_index_or_table_id,
 536                u32 flags,
 537                ip6_address_t * address,
 538                u32 address_length)
 539 {
 540   ip6_fib_t * fib = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 541   ip6_address_t masked_address;
 542   BVT(clib_bihash_kv) kv, value;
 543
 544   ASSERT (address_length < ARRAY_LEN (im->fib_masks));
 545   clib_memcpy (&masked_address, address, sizeof (masked_address));
 546   ip6_address_mask (&masked_address, &im->fib_masks[address_length]);
 547
 548   kv.key[0] = masked_address.as_u64[0];
 549   kv.key[1] = masked_address.as_u64[1];
 550   kv.key[2] = ((u64)((fib - im->fibs))<<32) | address_length;
 551
 552   if (BV(clib_bihash_search)(&im->ip6_lookup_table, &kv, &value) == 0)
 553     return (value.value);
 554   return 0;
 555 }
 556
 557 void
 558 ip6_foreach_matching_route (ip6_main_t * im,
 559                             u32 table_index_or_table_id,
 560                             u32 flags,
 561                             ip6_address_t * dst_address,
 562                             u32 address_length,
 563                             ip6_address_t ** results,
 564                             u8 ** result_lengths)
 565 {
 566   ip6_fib_t * fib =
 567     find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 568   BVT(clib_bihash) * h = &im->ip6_lookup_table;
 569   BVT(clib_bihash_value) * v;
 570   clib_bihash_bucket_t * b;
 571   int i, j, k;
 572
 573   if (*results)
 574     _vec_len (*results) = 0;
 575   if (*result_lengths)
 576     _vec_len (*result_lengths) = 0;
 577
 578   /* Walk the table looking for routes which match the supplied address */
 579   for (i = 0; i < h->nbuckets; i++)
 580     {
 581       b = &h->buckets [i];
 582       if (b->offset == 0)
 583           continue;
 584
 585       v = BV(clib_bihash_get_value) (h, b->offset);
 586       for (j = 0; j < (1<<b->log2_pages); j++)
 587         {
 588           for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
 589             {
 590               if (BV(clib_bihash_is_free)(&v->kvp[k]))
 591                 continue;
 592
 593               if ((v->kvp[k].key[2]
 594                    == (((u64)((fib - im->fibs))<<32) | address_length))
 595                   && ip6_destination_matches_route
 596                   (im, dst_address, (ip6_address_t *) &v->kvp[k],
 597                    address_length))
 598                 {
 599                   ip6_address_t * a;
 600
 601                   a = (ip6_address_t *)(&v->kvp[k]);
 602
 603                   vec_add1 (*results, a[0]);
 604                   vec_add1 (*result_lengths, address_length);
 605                 }
 606             }
 607           v++;
 608         }
 609     }
 610 }
 611
 612 void ip6_maybe_remap_adjacencies (ip6_main_t * im,
 613                                   u32 table_index_or_table_id,
 614                                   u32 flags)
 615 {
 616 #if SOONE
 617   ip6_fib_t * fib
 618     = find_ip6_fib_by_table_index_or_id (im, table_index_or_table_id, flags);
 619 #endif
 620   ip_lookup_main_t * lm = &im->lookup_main;
 621
 622   if (lm->n_adjacency_remaps == 0)
 623     return;
 624
 625   clib_warning ("unimplemented, please report to vpp-dev@cisco.com");
 626
 627   /* All remaps have been performed. */
 628   lm->n_adjacency_remaps = 0;
 629 }
 630
 631 void ip6_delete_matching_routes (ip6_main_t * im,
 632                                  u32 table_index_or_table_id,
 633                                  u32 flags,
 634                                  ip6_address_t * address,
 635                                  u32 address_length)
 636 {
 637   /* $$$$ static may be OK - this should happen only on thread 0 */
 638   static ip6_address_t * matching_addresses;
 639   static u8 * matching_address_lengths;
 640   u32 l, i;
 641   ip6_add_del_route_args_t a;
 642
 643   vlib_smp_unsafe_warning();
 644
 645   a.flags = IP6_ROUTE_FLAG_DEL | IP6_ROUTE_FLAG_NO_REDISTRIBUTE | flags;
 646   a.table_index_or_table_id = table_index_or_table_id;
 647   a.adj_index = ~0;
 648   a.add_adj = 0;
 649   a.n_add_adj = 0;
 650
 651   for (l = address_length + 1; l <= 128; l++)
 652     {
 653       ip6_foreach_matching_route (im, table_index_or_table_id, flags,
 654                                   address,
 655                                   l,
 656                                   &matching_addresses,
 657                                   &matching_address_lengths);
 658       for (i = 0; i < vec_len (matching_addresses); i++)
 659         {
 660           a.dst_address = matching_addresses[i];
 661           a.dst_address_length = matching_address_lengths[i];
 662           ip6_add_del_route (im, &a);
 663         }
 664     }
 665
 666   ip6_maybe_remap_adjacencies (im, table_index_or_table_id, flags);
 667 }
 668
 669 void
 670 ip6_forward_next_trace (vlib_main_t * vm,
 671                         vlib_node_runtime_t * node,
 672                         vlib_frame_t * frame,
 673                         vlib_rx_or_tx_t which_adj_index);
 674
 675 always_inline uword
 676 ip6_lookup_inline (vlib_main_t * vm,
 677                    vlib_node_runtime_t * node,
 678                    vlib_frame_t * frame,
 679                    int is_indirect)
 680 {
 681   ip6_main_t * im = &ip6_main;
 682   ip_lookup_main_t * lm = &im->lookup_main;
 683   vlib_combined_counter_main_t * cm = &im->lookup_main.adjacency_counters;
 684   u32 n_left_from, n_left_to_next, * from, * to_next;
 685   ip_lookup_next_t next;
 686   u32 cpu_index = os_get_cpu_number();
 687
 688   from = vlib_frame_vector_args (frame);
 689   n_left_from = frame->n_vectors;
 690   next = node->cached_next_index;
 691
 692   while (n_left_from > 0)
 693     {
 694       vlib_get_next_frame (vm, node, next,
 695                            to_next, n_left_to_next);
 696
 697       while (n_left_from >= 4 && n_left_to_next >= 2)
 698         {
 699           vlib_buffer_t * p0, * p1;
 700           u32 pi0, pi1, adj_index0, adj_index1, wrong_next;
 701           ip_lookup_next_t next0, next1;
 702           ip6_header_t * ip0, * ip1;
 703           ip_adjacency_t * adj0, * adj1;
 704           ip6_address_t * dst_addr0, * dst_addr1;
 705           u32 fib_index0, fib_index1;
 706           u32 flow_hash_config0, flow_hash_config1;
 707
 708           /* Prefetch next iteration. */
 709           {
 710             vlib_buffer_t * p2, * p3;
 711
 712             p2 = vlib_get_buffer (vm, from[2]);
 713             p3 = vlib_get_buffer (vm, from[3]);
 714
 715             vlib_prefetch_buffer_header (p2, LOAD);
 716             vlib_prefetch_buffer_header (p3, LOAD);
 717             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 718             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 719           }
 720
 721           pi0 = to_next[0] = from[0];
 722           pi1 = to_next[1] = from[1];
 723
 724           p0 = vlib_get_buffer (vm, pi0);
 725           p1 = vlib_get_buffer (vm, pi1);
 726
 727           ip0 = vlib_buffer_get_current (p0);
 728           ip1 = vlib_buffer_get_current (p1);
 729
 730           if (is_indirect)
 731             {
 732               ip_adjacency_t * iadj0, * iadj1;
 733               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 734               iadj1 = ip_get_adjacency (lm, vnet_buffer(p1)->ip.adj_index[VLIB_TX]);
 735               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 736               dst_addr1 = &iadj1->indirect.next_hop.ip6;
 737             }
 738           else
 739             {
 740               dst_addr0 = &ip0->dst_address;
 741               dst_addr1 = &ip1->dst_address;
 742             }
 743
 744           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 745           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 746
 747           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 748             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 749           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 750             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 751
 752           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 753           adj_index1 = ip6_fib_lookup_with_table (im, fib_index1, dst_addr1);
 754
 755           adj0 = ip_get_adjacency (lm, adj_index0);
 756           adj1 = ip_get_adjacency (lm, adj_index1);
 757
 758           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 759             {
 760               adj_index0 = ip6_fib_lookup_with_table
 761                 (im, adj0->explicit_fib_index, dst_addr0);
 762               adj0 = ip_get_adjacency (lm, adj_index0);
 763             }
 764           if (PREDICT_FALSE (adj1->explicit_fib_index != ~0))
 765             {
 766               adj_index1 = ip6_fib_lookup_with_table
 767                 (im, adj1->explicit_fib_index, dst_addr1);
 768               adj1 = ip_get_adjacency (lm, adj_index1);
 769             }
 770
 771           next0 = adj0->lookup_next_index;
 772           next1 = adj1->lookup_next_index;
 773
 774           /* Only process the HBH Option Header if explicitly configured to do so */
 775           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 776             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 777           next1 = (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 778             adj_index1 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj1->lookup_next_index;
 779
 780           vnet_buffer (p0)->ip.flow_hash =
 781             vnet_buffer(p1)->ip.flow_hash = 0;
 782
 783           if (PREDICT_FALSE(adj0->n_adj > 1))
 784             {
 785               flow_hash_config0 =
 786                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 787               vnet_buffer (p0)->ip.flow_hash =
 788                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 789             }
 790
 791           if (PREDICT_FALSE(adj1->n_adj > 1))
 792             {
 793               flow_hash_config1 =
 794                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 795
 796               vnet_buffer (p1)->ip.flow_hash =
 797                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 798             }
 799
 800           ASSERT (adj0->n_adj > 0);
 801           ASSERT (adj1->n_adj > 0);
 802           ASSERT (is_pow2 (adj0->n_adj));
 803           ASSERT (is_pow2 (adj1->n_adj));
 804           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 805           adj_index1 += (vnet_buffer (p1)->ip.flow_hash & (adj1->n_adj - 1));
 806
 807           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 808           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = adj_index1;
 809
 810           vlib_increment_combined_counter
 811               (cm, cpu_index, adj_index0, 1,
 812                vlib_buffer_length_in_chain (vm, p0));
 813           vlib_increment_combined_counter
 814               (cm, cpu_index, adj_index1, 1,
 815                vlib_buffer_length_in_chain (vm, p1));
 816
 817           from += 2;
 818           to_next += 2;
 819           n_left_to_next -= 2;
 820           n_left_from -= 2;
 821
 822           wrong_next = (next0 != next) + 2*(next1 != next);
 823           if (PREDICT_FALSE (wrong_next != 0))
 824             {
 825               switch (wrong_next)
 826                 {
 827                 case 1:
 828                   /* A B A */
 829                   to_next[-2] = pi1;
 830                   to_next -= 1;
 831                   n_left_to_next += 1;
 832                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 833                   break;
 834
 835                 case 2:
 836                   /* A A B */
 837                   to_next -= 1;
 838                   n_left_to_next += 1;
 839                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 840                   break;
 841
 842                 case 3:
 843                   /* A B C */
 844                   to_next -= 2;
 845                   n_left_to_next += 2;
 846                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 847                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 848                   if (next0 == next1)
 849                     {
 850                       /* A B B */
 851                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 852                       next = next1;
 853                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 854                     }
 855                 }
 856             }
 857         }
 858
 859       while (n_left_from > 0 && n_left_to_next > 0)
 860         {
 861           vlib_buffer_t * p0;
 862           ip6_header_t * ip0;
 863           u32 pi0, adj_index0;
 864           ip_lookup_next_t next0;
 865           ip_adjacency_t * adj0;
 866           ip6_address_t * dst_addr0;
 867           u32 fib_index0, flow_hash_config0;
 868
 869           pi0 = from[0];
 870           to_next[0] = pi0;
 871
 872           p0 = vlib_get_buffer (vm, pi0);
 873
 874           ip0 = vlib_buffer_get_current (p0);
 875
 876           if (is_indirect)
 877             {
 878               ip_adjacency_t * iadj0;
 879               iadj0 = ip_get_adjacency (lm, vnet_buffer(p0)->ip.adj_index[VLIB_TX]);
 880               dst_addr0 = &iadj0->indirect.next_hop.ip6;
 881             }
 882           else
 883             {
 884               dst_addr0 = &ip0->dst_address;
 885             }
 886
 887           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 888           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 889             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 890
 891           flow_hash_config0 =
 892               vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 893
 894           adj_index0 = ip6_fib_lookup_with_table (im, fib_index0, dst_addr0);
 895
 896           adj0 = ip_get_adjacency (lm, adj_index0);
 897
 898           if (PREDICT_FALSE (adj0->explicit_fib_index != ~0))
 899             {
 900               adj_index0 = ip6_fib_lookup_with_table
 901                 (im, adj0->explicit_fib_index, dst_addr0);
 902               adj0 = ip_get_adjacency (lm, adj_index0);
 903             }
 904
 905           /* Only process the HBH Option Header if explicitly configured to do so */
 906           next0 = (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) && im->hbh_enabled &&
 907             adj_index0 ? (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : adj0->lookup_next_index;
 908
 909           vnet_buffer (p0)->ip.flow_hash = 0;
 910
 911           if (PREDICT_FALSE(adj0->n_adj > 1))
 912             {
 913               flow_hash_config0 =
 914                 vec_elt_at_index (im->fibs,fib_index0)->flow_hash_config;
 915               vnet_buffer (p0)->ip.flow_hash =
 916                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 917             }
 918
 919           ASSERT (adj0->n_adj > 0);
 920           ASSERT (is_pow2 (adj0->n_adj));
 921           adj_index0 += (vnet_buffer (p0)->ip.flow_hash & (adj0->n_adj - 1));
 922
 923           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = adj_index0;
 924
 925           vlib_increment_combined_counter
 926               (cm, cpu_index, adj_index0, 1,
 927                vlib_buffer_length_in_chain (vm, p0));
 928
 929           from += 1;
 930           to_next += 1;
 931           n_left_to_next -= 1;
 932           n_left_from -= 1;
 933
 934           if (PREDICT_FALSE (next0 != next))
 935             {
 936               n_left_to_next += 1;
 937               vlib_put_next_frame (vm, node, next, n_left_to_next);
 938               next = next0;
 939               vlib_get_next_frame (vm, node, next,
 940                                    to_next, n_left_to_next);
 941               to_next[0] = pi0;
 942               to_next += 1;
 943               n_left_to_next -= 1;
 944             }
 945         }
 946
 947       vlib_put_next_frame (vm, node, next, n_left_to_next);
 948     }
 949
 950   if (node->flags & VLIB_NODE_FLAG_TRACE)
 951       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 952
 953   return frame->n_vectors;
 954 }
 955
 956 void ip6_adjacency_set_interface_route (vnet_main_t * vnm,
 957                                         ip_adjacency_t * adj,
 958                                         u32 sw_if_index,
 959                                         u32 if_address_index)
 960 {
 961   vnet_hw_interface_t * hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
 962   ip_lookup_next_t n;
 963   u32 node_index;
 964
 965   if (hw->hw_class_index == ethernet_hw_interface_class.index
 966       || hw->hw_class_index == srp_hw_interface_class.index)
 967     {
 968       n = IP_LOOKUP_NEXT_ARP;
 969       node_index = ip6_discover_neighbor_node.index;
 970       adj->if_address_index = if_address_index;
 971       adj->arp.next_hop.ip6.as_u64[0] = 0;
 972       adj->arp.next_hop.ip6.as_u64[1] = 0;
 973   }
 974   else
 975     {
 976       n = IP_LOOKUP_NEXT_REWRITE;
 977       node_index = ip6_rewrite_node.index;
 978     }
 979
 980  adj->lookup_next_index = n;
 981  adj->explicit_fib_index = ~0;
 982
 983  vnet_rewrite_for_sw_interface
 984    (vnm,
 985     VNET_L3_PACKET_TYPE_IP6,
 986     sw_if_index,
 987     node_index,
 988     VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST,
 989     &adj->rewrite_header,
 990     sizeof (adj->rewrite_data));
 991 }
 992
 993 static void
 994 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 995                           ip6_main_t * im, u32 fib_index,
 996                           ip_interface_address_t * a)
 997 {
 998   ip_lookup_main_t * lm = &im->lookup_main;
 999   ip_adjacency_t * adj;
1000   ip6_address_t * address = ip_interface_address_get_address (lm, a);
1001   ip6_add_del_route_args_t x;
1002   vnet_hw_interface_t * hw_if = vnet_get_sup_hw_interface (vnm, sw_if_index);
1003   u32 classify_table_index;
1004
1005   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1006   x.table_index_or_table_id = fib_index;
1007   x.flags = (IP6_ROUTE_FLAG_ADD
1008              | IP6_ROUTE_FLAG_FIB_INDEX
1009              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1010   x.dst_address = address[0];
1011   x.dst_address_length = a->address_length;
1012   x.n_add_adj = 0;
1013   x.add_adj = 0;
1014
1015   a->neighbor_probe_adj_index = ~0;
1016   if (a->address_length < 128)
1017     {
1018       adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1019                               &x.adj_index);
1020       ip6_adjacency_set_interface_route (vnm, adj, sw_if_index, a - lm->if_address_pool);
1021       ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1022       ip6_add_del_route (im, &x);
1023       a->neighbor_probe_adj_index = x.adj_index;
1024     }
1025
1026   /* Add e.g. ::1/128 as local to this host. */
1027   adj = ip_add_adjacency (lm, /* template */ 0, /* block size */ 1,
1028                           &x.adj_index);
1029
1030   classify_table_index = ~0;
1031   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
1032     classify_table_index = lm->classify_table_index_by_sw_if_index [sw_if_index];
1033   if (classify_table_index != (u32) ~0)
1034     {
1035       adj->lookup_next_index = IP_LOOKUP_NEXT_CLASSIFY;
1036       adj->classify.table_index = classify_table_index;
1037     }
1038   else
1039     adj->lookup_next_index = IP_LOOKUP_NEXT_LOCAL;
1040
1041   adj->if_address_index = a - lm->if_address_pool;
1042   adj->rewrite_header.sw_if_index = sw_if_index;
1043   adj->rewrite_header.max_l3_packet_bytes = hw_if->max_l3_packet_bytes[VLIB_RX];
1044   adj->rewrite_header.data_bytes = 0;
1045   ip_call_add_del_adjacency_callbacks (lm, x.adj_index, /* is_del */ 0);
1046   x.dst_address_length = 128;
1047   ip6_add_del_route (im, &x);
1048 }
1049
1050 static void
1051 ip6_del_interface_routes (ip6_main_t * im, u32 fib_index,
1052                           ip6_address_t * address, u32 address_length)
1053 {
1054   ip6_add_del_route_args_t x;
1055
1056   /* Add e.g. 1.0.0.0/8 as interface route (arp for Ethernet). */
1057   x.table_index_or_table_id = fib_index;
1058   x.flags = (IP6_ROUTE_FLAG_DEL
1059              | IP6_ROUTE_FLAG_FIB_INDEX
1060              | IP6_ROUTE_FLAG_NO_REDISTRIBUTE);
1061   x.dst_address = address[0];
1062   x.dst_address_length = address_length;
1063   x.adj_index = ~0;
1064   x.n_add_adj = 0;
1065   x.add_adj = 0;
1066
1067   if (address_length < 128)
1068     {
1069       /* Don't wipe out fe80::0/64 */
1070       if (address_length != 64 ||
1071           address[0].as_u64[0] != clib_net_to_host_u64(0xfe80000000000000ULL))
1072         ip6_add_del_route (im, &x);
1073     }
1074
1075   x.dst_address_length = 128;
1076   ip6_add_del_route (im, &x);
1077
1078   ip6_delete_matching_routes (im,
1079                               fib_index,
1080                               IP6_ROUTE_FLAG_FIB_INDEX,
1081                               address,
1082                               address_length);
1083 }
1084
1085 typedef struct {
1086     u32 sw_if_index;
1087     ip6_address_t address;
1088     u32 length;
1089 } ip6_interface_address_t;
1090
1091 static clib_error_t *
1092 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1093                                         u32 sw_if_index,
1094                                         ip6_address_t * new_address,
1095                                         u32 new_length,
1096                                         u32 redistribute,
1097                                         u32 insert_routes,
1098                                         u32 is_del);
1099
1100 static clib_error_t *
1101 ip6_add_del_interface_address_internal (vlib_main_t * vm,
1102                                         u32 sw_if_index,
1103                                         ip6_address_t * address,
1104                                         u32 address_length,
1105                                         u32 redistribute,
1106                                         u32 insert_routes,
1107                                         u32 is_del)
1108 {
1109   vnet_main_t * vnm = vnet_get_main();
1110   ip6_main_t * im = &ip6_main;
1111   ip_lookup_main_t * lm = &im->lookup_main;
1112   clib_error_t * error;
1113   u32 if_address_index;
1114   ip6_address_fib_t ip6_af, * addr_fib = 0;
1115
1116   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1117   ip6_addr_fib_init (&ip6_af, address,
1118                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
1119   vec_add1 (addr_fib, ip6_af);
1120
1121   {
1122     uword elts_before = pool_elts (lm->if_address_pool);
1123
1124     error = ip_interface_address_add_del
1125       (lm,
1126        sw_if_index,
1127        addr_fib,
1128        address_length,
1129        is_del,
1130        &if_address_index);
1131     if (error)
1132       goto done;
1133
1134     /* Pool did not grow: add duplicate address. */
1135     if (elts_before == pool_elts (lm->if_address_pool))
1136       goto done;
1137   }
1138
1139   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index) && insert_routes)
1140     {
1141       if (is_del)
1142         ip6_del_interface_routes (im, ip6_af.fib_index, address,
1143                                   address_length);
1144
1145       else
1146         ip6_add_interface_routes (vnm, sw_if_index,
1147                                   im, ip6_af.fib_index,
1148                                   pool_elt_at_index (lm->if_address_pool, if_address_index));
1149     }
1150
1151   {
1152     ip6_add_del_interface_address_callback_t * cb;
1153     vec_foreach (cb, im->add_del_interface_address_callbacks)
1154       cb->function (im, cb->function_opaque, sw_if_index,
1155                     address, address_length,
1156                     if_address_index,
1157                     is_del);
1158   }
1159
1160  done:
1161   vec_free (addr_fib);
1162   return error;
1163 }
1164
1165 clib_error_t *
1166 ip6_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
1167                                ip6_address_t * address, u32 address_length,
1168                                u32 is_del)
1169 {
1170   return ip6_add_del_interface_address_internal
1171     (vm, sw_if_index, address, address_length,
1172      /* redistribute */ 1,
1173      /* insert_routes */ 1,
1174      is_del);
1175 }
1176
1177 clib_error_t *
1178 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
1179                                 u32 sw_if_index,
1180                                 u32 flags)
1181 {
1182   ip6_main_t * im = &ip6_main;
1183   ip_interface_address_t * ia;
1184   ip6_address_t * a;
1185   u32 is_admin_up, fib_index;
1186
1187   /* Fill in lookup tables with default table (0). */
1188   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1189
1190   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
1191
1192   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1193
1194   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
1195
1196   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
1197                                 0 /* honor unnumbered */,
1198   ({
1199     a = ip_interface_address_get_address (&im->lookup_main, ia);
1200     if (is_admin_up)
1201       ip6_add_interface_routes (vnm, sw_if_index,
1202                                 im, fib_index,
1203                                 ia);
1204     else
1205       ip6_del_interface_routes (im, fib_index,
1206                                 a, ia->address_length);
1207   }));
1208
1209   return 0;
1210 }
1211
1212 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
1213
1214 /* Built-in ip6 unicast rx feature path definition */
1215 VNET_IP6_UNICAST_FEATURE_INIT (ip6_inacl, static) = {
1216   .node_name = "ip6-inacl",
1217   .runs_before = {"ipsec-input-ip6", 0},
1218   .feature_index = &ip6_main.ip6_unicast_rx_feature_check_access,
1219 };
1220
1221 VNET_IP6_UNICAST_FEATURE_INIT (ip6_ipsec, static) = {
1222   .node_name = "ipsec-input-ip6",
1223   .runs_before = {"l2tp-decap", 0},
1224   .feature_index = &ip6_main.ip6_unicast_rx_feature_ipsec,
1225 };
1226
1227 VNET_IP6_UNICAST_FEATURE_INIT (ip6_l2tp, static) = {
1228   .node_name = "l2tp-decap",
1229   .runs_before = {"vpath-input-ip6", 0},
1230   .feature_index = &ip6_main.ip6_unicast_rx_feature_l2tp_decap,
1231 };
1232
1233 VNET_IP6_UNICAST_FEATURE_INIT (ip6_vpath, static) = {
1234   .node_name = "vpath-input-ip6",
1235   .runs_before = {"ip6-lookup", 0},
1236   .feature_index = &ip6_main.ip6_unicast_rx_feature_vpath,
1237 };
1238
1239 VNET_IP6_UNICAST_FEATURE_INIT (ip6_lookup, static) = {
1240   .node_name = "ip6-lookup",
1241   .runs_before = {0}, /* not before any other features */
1242   .feature_index = &ip6_main.ip6_unicast_rx_feature_lookup,
1243 };
1244
1245 /* Built-in ip6 multicast rx feature path definition (none now) */
1246 VNET_IP6_MULTICAST_FEATURE_INIT (ip4_vpath_mc, static) = {
1247   .node_name = "vpath-input-ip6",
1248   .runs_before = {"ip6-lookup", 0},
1249   .feature_index = &ip6_main.ip6_multicast_rx_feature_vpath,
1250 };
1251
1252 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_lookup, static) = {
1253   .node_name = "ip6-lookup",
1254   .runs_before = {0}, /* not before any other features */
1255   .feature_index = &ip6_main.ip6_multicast_rx_feature_lookup,
1256 };
1257
1258 static char * feature_start_nodes[] =
1259   {"ip6-input"};
1260
1261 static clib_error_t *
1262 ip6_feature_init (vlib_main_t * vm, ip6_main_t * im)
1263 {
1264   ip_lookup_main_t * lm = &im->lookup_main;
1265   clib_error_t * error;
1266   vnet_cast_t cast;
1267
1268   for (cast = 0; cast < VNET_N_CAST; cast++)
1269     {
1270       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1271       vnet_config_main_t * vcm = &cm->config_main;
1272
1273       if ((error = ip_feature_init_cast (vm, cm, vcm,
1274                                          feature_start_nodes,
1275                                          ARRAY_LEN(feature_start_nodes),
1276                                          cast,
1277                                          0 /* is_ip4 */)))
1278         return error;
1279     }
1280   return 0;
1281 }
1282
1283 clib_error_t *
1284 ip6_sw_interface_add_del (vnet_main_t * vnm,
1285                           u32 sw_if_index,
1286                           u32 is_add)
1287 {
1288   vlib_main_t * vm = vnm->vlib_main;
1289   ip6_main_t * im = &ip6_main;
1290   ip_lookup_main_t * lm = &im->lookup_main;
1291   u32 ci, cast;
1292   u32 feature_index;
1293
1294   for (cast = 0; cast < VNET_N_CAST; cast++)
1295     {
1296       ip_config_main_t * cm = &lm->rx_config_mains[cast];
1297       vnet_config_main_t * vcm = &cm->config_main;
1298
1299       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
1300       ci = cm->config_index_by_sw_if_index[sw_if_index];
1301
1302       if (cast == VNET_UNICAST)
1303         feature_index = im->ip6_unicast_rx_feature_lookup;
1304       else
1305         feature_index = im->ip6_multicast_rx_feature_lookup;
1306
1307       if (is_add)
1308         ci = vnet_config_add_feature (vm, vcm,
1309                                       ci,
1310                                       feature_index,
1311                                       /* config data */ 0,
1312                                       /* # bytes of config data */ 0);
1313       else
1314         ci = vnet_config_del_feature (vm, vcm,
1315                                       ci,
1316                                       feature_index,
1317                                       /* config data */ 0,
1318                                       /* # bytes of config data */ 0);
1319
1320       cm->config_index_by_sw_if_index[sw_if_index] = ci;
1321     }
1322   return /* no error */ 0;
1323 }
1324
1325 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
1326
1327 static uword
1328 ip6_lookup (vlib_main_t * vm,
1329             vlib_node_runtime_t * node,
1330             vlib_frame_t * frame)
1331 {
1332   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 0);
1333 }
1334
1335 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
1336
1337 VLIB_REGISTER_NODE (ip6_lookup_node) = {
1338   .function = ip6_lookup,
1339   .name = "ip6-lookup",
1340   .vector_size = sizeof (u32),
1341
1342   .format_trace = format_ip6_lookup_trace,
1343
1344   .n_next_nodes = IP6_LOOKUP_N_NEXT,
1345   .next_nodes = IP6_LOOKUP_NEXT_NODES,
1346 };
1347
1348 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
1349
1350 static uword
1351 ip6_indirect (vlib_main_t * vm,
1352               vlib_node_runtime_t * node,
1353               vlib_frame_t * frame)
1354 {
1355   return ip6_lookup_inline (vm, node, frame, /* is_indirect */ 1);
1356 }
1357
1358
1359 VLIB_REGISTER_NODE (ip6_indirect_node) = {
1360   .function = ip6_indirect,
1361   .name = "ip6-indirect",
1362   .vector_size = sizeof (u32),
1363   .sibling_of = "ip6-lookup",
1364   .format_trace = format_ip6_lookup_trace,
1365   .n_next_nodes = 0,
1366 };
1367
1368 VLIB_NODE_FUNCTION_MULTIARCH (ip6_indirect_node, ip6_indirect)
1369
1370 typedef struct {
1371   /* Adjacency taken. */
1372   u32 adj_index;
1373   u32 flow_hash;
1374   u32 fib_index;
1375
1376   /* Packet data, possibly *after* rewrite. */
1377   u8 packet_data[128 - 1*sizeof(u32)];
1378 } ip6_forward_next_trace_t;
1379
1380 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
1381 {
1382   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1383   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1384   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1385   uword indent = format_get_indent (s);
1386
1387   s = format(s, "%U%U",
1388              format_white_space, indent,
1389              format_ip6_header, t->packet_data);
1390   return s;
1391 }
1392
1393 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
1394 {
1395   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1396   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1397   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1398   vnet_main_t * vnm = vnet_get_main();
1399   ip6_main_t * im = &ip6_main;
1400   uword indent = format_get_indent (s);
1401
1402   s = format (s, "fib %d adj-idx %d : %U flow hash: 0x%08x",
1403               t->fib_index, t->adj_index, format_ip_adjacency,
1404               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1405   s = format(s, "\n%U%U",
1406              format_white_space, indent,
1407              format_ip6_header, t->packet_data);
1408   return s;
1409 }
1410
1411
1412 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
1413 {
1414   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1415   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1416   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
1417   vnet_main_t * vnm = vnet_get_main();
1418   ip6_main_t * im = &ip6_main;
1419   uword indent = format_get_indent (s);
1420
1421   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
1422               t->fib_index, t->adj_index, format_ip_adjacency,
1423               vnm, &im->lookup_main, t->adj_index, t->flow_hash);
1424   s = format (s, "\n%U%U",
1425               format_white_space, indent,
1426               format_ip_adjacency_packet_data,
1427               vnm, &im->lookup_main, t->adj_index,
1428               t->packet_data, sizeof (t->packet_data));
1429   return s;
1430 }
1431
1432 /* Common trace function for all ip6-forward next nodes. */
1433 void
1434 ip6_forward_next_trace (vlib_main_t * vm,
1435                         vlib_node_runtime_t * node,
1436                         vlib_frame_t * frame,
1437                         vlib_rx_or_tx_t which_adj_index)
1438 {
1439   u32 * from, n_left;
1440   ip6_main_t * im = &ip6_main;
1441
1442   n_left = frame->n_vectors;
1443   from = vlib_frame_vector_args (frame);
1444
1445   while (n_left >= 4)
1446     {
1447       u32 bi0, bi1;
1448       vlib_buffer_t * b0, * b1;
1449       ip6_forward_next_trace_t * t0, * t1;
1450
1451       /* Prefetch next iteration. */
1452       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1453       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1454
1455       bi0 = from[0];
1456       bi1 = from[1];
1457
1458       b0 = vlib_get_buffer (vm, bi0);
1459       b1 = vlib_get_buffer (vm, bi1);
1460
1461       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1462         {
1463           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1464           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1465           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1466           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1467               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1468               vec_elt (im->fib_index_by_sw_if_index,
1469                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1470
1471           clib_memcpy (t0->packet_data,
1472                   vlib_buffer_get_current (b0),
1473                   sizeof (t0->packet_data));
1474         }
1475       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1476         {
1477           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1478           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1479           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1480           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1481               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1482               vec_elt (im->fib_index_by_sw_if_index,
1483                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1484
1485           clib_memcpy (t1->packet_data,
1486                   vlib_buffer_get_current (b1),
1487                   sizeof (t1->packet_data));
1488         }
1489       from += 2;
1490       n_left -= 2;
1491     }
1492
1493   while (n_left >= 1)
1494     {
1495       u32 bi0;
1496       vlib_buffer_t * b0;
1497       ip6_forward_next_trace_t * t0;
1498
1499       bi0 = from[0];
1500
1501       b0 = vlib_get_buffer (vm, bi0);
1502
1503       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1504         {
1505           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1506           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1507           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1508           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1509               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1510               vec_elt (im->fib_index_by_sw_if_index,
1511                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1512
1513           clib_memcpy (t0->packet_data,
1514                   vlib_buffer_get_current (b0),
1515                   sizeof (t0->packet_data));
1516         }
1517       from += 1;
1518       n_left -= 1;
1519     }
1520 }
1521
1522 static uword
1523 ip6_drop_or_punt (vlib_main_t * vm,
1524                   vlib_node_runtime_t * node,
1525                   vlib_frame_t * frame,
1526                   ip6_error_t error_code)
1527 {
1528   u32 * buffers = vlib_frame_vector_args (frame);
1529   uword n_packets = frame->n_vectors;
1530
1531   vlib_error_drop_buffers (vm, node,
1532                            buffers,
1533                            /* stride */ 1,
1534                            n_packets,
1535                            /* next */ 0,
1536                            ip6_input_node.index,
1537                            error_code);
1538
1539   if (node->flags & VLIB_NODE_FLAG_TRACE)
1540     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1541
1542   return n_packets;
1543 }
1544
1545 static uword
1546 ip6_drop (vlib_main_t * vm,
1547           vlib_node_runtime_t * node,
1548           vlib_frame_t * frame)
1549 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1550
1551 static uword
1552 ip6_punt (vlib_main_t * vm,
1553           vlib_node_runtime_t * node,
1554           vlib_frame_t * frame)
1555 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1556
1557 static uword
1558 ip6_miss (vlib_main_t * vm,
1559           vlib_node_runtime_t * node,
1560           vlib_frame_t * frame)
1561 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_DST_LOOKUP_MISS); }
1562
1563 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1564   .function = ip6_drop,
1565   .name = "ip6-drop",
1566   .vector_size = sizeof (u32),
1567
1568   .format_trace = format_ip6_forward_next_trace,
1569
1570   .n_next_nodes = 1,
1571   .next_nodes = {
1572     [0] = "error-drop",
1573   },
1574 };
1575
1576 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1577
1578 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1579   .function = ip6_punt,
1580   .name = "ip6-punt",
1581   .vector_size = sizeof (u32),
1582
1583   .format_trace = format_ip6_forward_next_trace,
1584
1585   .n_next_nodes = 1,
1586   .next_nodes = {
1587     [0] = "error-punt",
1588   },
1589 };
1590
1591 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1592
1593 VLIB_REGISTER_NODE (ip6_miss_node,static) = {
1594   .function = ip6_miss,
1595   .name = "ip6-miss",
1596   .vector_size = sizeof (u32),
1597
1598   .format_trace = format_ip6_forward_next_trace,
1599
1600   .n_next_nodes = 1,
1601   .next_nodes = {
1602     [0] = "error-drop",
1603   },
1604 };
1605
1606 VLIB_NODE_FUNCTION_MULTIARCH (ip6_miss_node, ip6_miss)
1607
1608 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1609   .function = ip6_drop,
1610   .name = "ip6-multicast",
1611   .vector_size = sizeof (u32),
1612
1613   .format_trace = format_ip6_forward_next_trace,
1614
1615   .n_next_nodes = 1,
1616   .next_nodes = {
1617     [0] = "error-drop",
1618   },
1619 };
1620
1621 /* Compute TCP/UDP/ICMP6 checksum in software. */
1622 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1623 {
1624   ip_csum_t sum0;
1625   u16 sum16, payload_length_host_byte_order;
1626   u32 i, n_this_buffer, n_bytes_left;
1627   u32 headers_size = sizeof(ip0[0]);
1628   void * data_this_buffer;
1629
1630   ASSERT(bogus_lengthp);
1631   *bogus_lengthp = 0;
1632
1633   /* Initialize checksum with ip header. */
1634   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1635   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1636   data_this_buffer = (void *) (ip0 + 1);
1637
1638   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1639     {
1640       sum0 = ip_csum_with_carry (sum0,
1641                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1642       sum0 = ip_csum_with_carry (sum0,
1643                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1644     }
1645
1646   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1647   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1648     {
1649       u32  skip_bytes;
1650       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1651
1652       /* validate really icmp6 next */
1653       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1654
1655       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1656       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1657
1658       payload_length_host_byte_order  -= skip_bytes;
1659       headers_size += skip_bytes;
1660    }
1661
1662   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1663 #if DPDK > 0
1664   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1665   {
1666     struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
1667     u8 nb_segs = mb->nb_segs;
1668
1669     n_this_buffer = (p0->current_length > headers_size ?
1670                      p0->current_length - headers_size : 0);
1671     while (n_bytes_left)
1672       {
1673         sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1674         n_bytes_left -= n_this_buffer;
1675
1676         mb = mb->next;
1677         nb_segs--;
1678         if ((nb_segs == 0) || (mb == 0))
1679           break;
1680
1681         data_this_buffer = rte_ctrlmbuf_data(mb);
1682         n_this_buffer = mb->data_len;
1683       }
1684     if (n_bytes_left || nb_segs)
1685       {
1686         *bogus_lengthp = 1;
1687         return 0xfefe;
1688       }
1689   }
1690   else sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1691 #else
1692   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1693     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1694   while (1)
1695     {
1696       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1697       n_bytes_left -= n_this_buffer;
1698       if (n_bytes_left == 0)
1699         break;
1700
1701       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1702         {
1703           *bogus_lengthp = 1;
1704           return 0xfefe;
1705         }
1706       p0 = vlib_get_buffer (vm, p0->next_buffer);
1707       data_this_buffer = vlib_buffer_get_current (p0);
1708       n_this_buffer = p0->current_length;
1709     }
1710 #endif /* DPDK */
1711
1712   sum16 = ~ ip_csum_fold (sum0);
1713
1714   return sum16;
1715 }
1716
1717 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1718 {
1719   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1720   udp_header_t * udp0;
1721   u16 sum16;
1722   int bogus_length;
1723
1724   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1725   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1726           || ip0->protocol == IP_PROTOCOL_ICMP6
1727           || ip0->protocol == IP_PROTOCOL_UDP
1728           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1729
1730   udp0 = (void *) (ip0 + 1);
1731   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1732     {
1733       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1734                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1735       return p0->flags;
1736     }
1737
1738   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1739
1740   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1741                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1742
1743   return p0->flags;
1744 }
1745
1746 static uword
1747 ip6_local (vlib_main_t * vm,
1748            vlib_node_runtime_t * node,
1749            vlib_frame_t * frame)
1750 {
1751   ip6_main_t * im = &ip6_main;
1752   ip_lookup_main_t * lm = &im->lookup_main;
1753   ip_local_next_t next_index;
1754   u32 * from, * to_next, n_left_from, n_left_to_next;
1755   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1756
1757   from = vlib_frame_vector_args (frame);
1758   n_left_from = frame->n_vectors;
1759   next_index = node->cached_next_index;
1760
1761   if (node->flags & VLIB_NODE_FLAG_TRACE)
1762     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1763
1764   while (n_left_from > 0)
1765     {
1766       vlib_get_next_frame (vm, node, next_index,
1767                            to_next, n_left_to_next);
1768
1769       while (n_left_from >= 4 && n_left_to_next >= 2)
1770         {
1771           vlib_buffer_t * p0, * p1;
1772           ip6_header_t * ip0, * ip1;
1773           udp_header_t * udp0, * udp1;
1774           u32 pi0, ip_len0, udp_len0, flags0, next0;
1775           u32 pi1, ip_len1, udp_len1, flags1, next1;
1776           i32 len_diff0, len_diff1;
1777           u8 error0, type0, good_l4_checksum0;
1778           u8 error1, type1, good_l4_checksum1;
1779
1780           pi0 = to_next[0] = from[0];
1781           pi1 = to_next[1] = from[1];
1782           from += 2;
1783           n_left_from -= 2;
1784           to_next += 2;
1785           n_left_to_next -= 2;
1786
1787           p0 = vlib_get_buffer (vm, pi0);
1788           p1 = vlib_get_buffer (vm, pi1);
1789
1790           ip0 = vlib_buffer_get_current (p0);
1791           ip1 = vlib_buffer_get_current (p1);
1792
1793           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1794           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1795
1796           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1797           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1798
1799           flags0 = p0->flags;
1800           flags1 = p1->flags;
1801
1802           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1803           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1804
1805           udp0 = ip6_next_header (ip0);
1806           udp1 = ip6_next_header (ip1);
1807
1808           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1809           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1810           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1811
1812           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1813           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1814
1815           /* Verify UDP length. */
1816           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1817           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1818           udp_len0 = clib_net_to_host_u16 (udp0->length);
1819           udp_len1 = clib_net_to_host_u16 (udp1->length);
1820
1821           len_diff0 = ip_len0 - udp_len0;
1822           len_diff1 = ip_len1 - udp_len1;
1823
1824           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1825           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1826
1827           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1828                              && ! good_l4_checksum0
1829                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1830             {
1831               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1832               good_l4_checksum0 =
1833                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1834             }
1835           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1836                              && ! good_l4_checksum1
1837                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1838             {
1839               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1840               good_l4_checksum1 =
1841                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1842             }
1843
1844           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1845
1846           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1847           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1848
1849           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1850           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1851           error0 = (! good_l4_checksum0
1852                     ? IP6_ERROR_UDP_CHECKSUM + type0
1853                     : error0);
1854           error1 = (! good_l4_checksum1
1855                     ? IP6_ERROR_UDP_CHECKSUM + type1
1856                     : error1);
1857
1858           /* Drop packets from unroutable hosts. */
1859           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1860           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1861             {
1862               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1863               error0 = (lm->miss_adj_index == src_adj_index0
1864                         ? IP6_ERROR_SRC_LOOKUP_MISS
1865                         : error0);
1866             }
1867           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL && type1 != IP_BUILTIN_PROTOCOL_ICMP)
1868             {
1869               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1870               error1 = (lm->miss_adj_index == src_adj_index1
1871                         ? IP6_ERROR_SRC_LOOKUP_MISS
1872                         : error1);
1873             }
1874
1875           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1876           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1877
1878           p0->error = error_node->errors[error0];
1879           p1->error = error_node->errors[error1];
1880
1881           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1882                                            to_next, n_left_to_next,
1883                                            pi0, pi1, next0, next1);
1884         }
1885
1886       while (n_left_from > 0 && n_left_to_next > 0)
1887         {
1888           vlib_buffer_t * p0;
1889           ip6_header_t * ip0;
1890           udp_header_t * udp0;
1891           u32 pi0, ip_len0, udp_len0, flags0, next0;
1892           i32 len_diff0;
1893           u8 error0, type0, good_l4_checksum0;
1894
1895           pi0 = to_next[0] = from[0];
1896           from += 1;
1897           n_left_from -= 1;
1898           to_next += 1;
1899           n_left_to_next -= 1;
1900
1901           p0 = vlib_get_buffer (vm, pi0);
1902
1903           ip0 = vlib_buffer_get_current (p0);
1904
1905           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1906           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1907
1908           flags0 = p0->flags;
1909
1910           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1911
1912           udp0 = ip6_next_header (ip0);
1913
1914           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1915           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1916
1917           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1918
1919           /* Verify UDP length. */
1920           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1921           udp_len0 = clib_net_to_host_u16 (udp0->length);
1922
1923           len_diff0 = ip_len0 - udp_len0;
1924
1925           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1926
1927           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1928                              && ! good_l4_checksum0
1929                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1930             {
1931               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1932               good_l4_checksum0 =
1933                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1934             }
1935
1936           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1937
1938           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1939
1940           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1941           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1942           error0 = (! good_l4_checksum0
1943                     ? IP6_ERROR_UDP_CHECKSUM + type0
1944                     : error0);
1945
1946           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1947           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL && type0 != IP_BUILTIN_PROTOCOL_ICMP)
1948             {
1949               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1950               error0 = (lm->miss_adj_index == src_adj_index0
1951                         ? IP6_ERROR_SRC_LOOKUP_MISS
1952                         : error0);
1953             }
1954
1955           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1956
1957           p0->error = error_node->errors[error0];
1958
1959           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1960                                            to_next, n_left_to_next,
1961                                            pi0, next0);
1962         }
1963
1964       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1965     }
1966
1967   return frame->n_vectors;
1968 }
1969
1970 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1971   .function = ip6_local,
1972   .name = "ip6-local",
1973   .vector_size = sizeof (u32),
1974
1975   .format_trace = format_ip6_forward_next_trace,
1976
1977   .n_next_nodes = IP_LOCAL_N_NEXT,
1978   .next_nodes = {
1979     [IP_LOCAL_NEXT_DROP] = "error-drop",
1980     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1981     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1982     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1983   },
1984 };
1985
1986 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
1987
1988 void ip6_register_protocol (u32 protocol, u32 node_index)
1989 {
1990   vlib_main_t * vm = vlib_get_main();
1991   ip6_main_t * im = &ip6_main;
1992   ip_lookup_main_t * lm = &im->lookup_main;
1993
1994   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1995   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1996 }
1997
1998 typedef enum {
1999   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2000   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
2001   IP6_DISCOVER_NEIGHBOR_N_NEXT,
2002 } ip6_discover_neighbor_next_t;
2003
2004 typedef enum {
2005   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
2006   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
2007   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
2008 } ip6_discover_neighbor_error_t;
2009
2010 static uword
2011 ip6_discover_neighbor (vlib_main_t * vm,
2012                        vlib_node_runtime_t * node,
2013                        vlib_frame_t * frame)
2014 {
2015   vnet_main_t * vnm = vnet_get_main();
2016   ip6_main_t * im = &ip6_main;
2017   ip_lookup_main_t * lm = &im->lookup_main;
2018   u32 * from, * to_next_drop;
2019   uword n_left_from, n_left_to_next_drop;
2020   static f64 time_last_seed_change = -1e100;
2021   static u32 hash_seeds[3];
2022   static uword hash_bitmap[256 / BITS (uword)];
2023   f64 time_now;
2024   int bogus_length;
2025
2026   if (node->flags & VLIB_NODE_FLAG_TRACE)
2027     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2028
2029   time_now = vlib_time_now (vm);
2030   if (time_now - time_last_seed_change > 1e-3)
2031     {
2032       uword i;
2033       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
2034                                              sizeof (hash_seeds));
2035       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
2036         hash_seeds[i] = r[i];
2037
2038       /* Mark all hash keys as been not-seen before. */
2039       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
2040         hash_bitmap[i] = 0;
2041
2042       time_last_seed_change = time_now;
2043     }
2044
2045   from = vlib_frame_vector_args (frame);
2046   n_left_from = frame->n_vectors;
2047
2048   while (n_left_from > 0)
2049     {
2050       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2051                            to_next_drop, n_left_to_next_drop);
2052
2053       while (n_left_from > 0 && n_left_to_next_drop > 0)
2054         {
2055           vlib_buffer_t * p0;
2056           ip6_header_t * ip0;
2057           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
2058           uword bm0;
2059           ip_adjacency_t * adj0;
2060           vnet_hw_interface_t * hw_if0;
2061           u32 next0;
2062
2063           pi0 = from[0];
2064
2065           p0 = vlib_get_buffer (vm, pi0);
2066
2067           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2068
2069           ip0 = vlib_buffer_get_current (p0);
2070
2071           adj0 = ip_get_adjacency (lm, adj_index0);
2072
2073           if (adj0->arp.next_hop.ip6.as_u64[0] ||
2074               adj0->arp.next_hop.ip6.as_u64[1]) {
2075             ip0->dst_address.as_u64[0] = adj0->arp.next_hop.ip6.as_u64[0];
2076             ip0->dst_address.as_u64[1] = adj0->arp.next_hop.ip6.as_u64[1];
2077           }
2078
2079           a0 = hash_seeds[0];
2080           b0 = hash_seeds[1];
2081           c0 = hash_seeds[2];
2082
2083           sw_if_index0 = adj0->rewrite_header.sw_if_index;
2084           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
2085
2086           a0 ^= sw_if_index0;
2087           b0 ^= ip0->dst_address.as_u32[0];
2088           c0 ^= ip0->dst_address.as_u32[1];
2089
2090           hash_v3_mix32 (a0, b0, c0);
2091
2092           b0 ^= ip0->dst_address.as_u32[2];
2093           c0 ^= ip0->dst_address.as_u32[3];
2094
2095           hash_v3_finalize32 (a0, b0, c0);
2096
2097           c0 &= BITS (hash_bitmap) - 1;
2098           c0 = c0 / BITS (uword);
2099           m0 = (uword) 1 << (c0 % BITS (uword));
2100
2101           bm0 = hash_bitmap[c0];
2102           drop0 = (bm0 & m0) != 0;
2103
2104           /* Mark it as seen. */
2105           hash_bitmap[c0] = bm0 | m0;
2106
2107           from += 1;
2108           n_left_from -= 1;
2109           to_next_drop[0] = pi0;
2110           to_next_drop += 1;
2111           n_left_to_next_drop -= 1;
2112
2113           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2114
2115           /* If the interface is link-down, drop the pkt */
2116           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
2117             drop0 = 1;
2118
2119           p0->error =
2120             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
2121                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
2122           if (drop0)
2123             continue;
2124
2125           {
2126             u32 bi0 = 0;
2127             icmp6_neighbor_solicitation_header_t * h0;
2128             vlib_buffer_t * b0;
2129
2130             h0 = vlib_packet_template_get_packet
2131               (vm, &im->discover_neighbor_packet_template, &bi0);
2132
2133             /*
2134              * Build ethernet header.
2135              * Choose source address based on destination lookup
2136              * adjacency.
2137              */
2138             if (ip6_src_address_for_packet (im, p0, &h0->ip.src_address,
2139                                                 sw_if_index0)) {
2140                 //There is no address on the interface
2141                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
2142                 vlib_buffer_free(vm, &bi0, 1);
2143                 continue;
2144             }
2145
2146             /*
2147              * Destination address is a solicited node multicast address.
2148              * We need to fill in
2149              * the low 24 bits with low 24 bits of target's address.
2150              */
2151             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
2152             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
2153             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
2154
2155             h0->neighbor.target_address = ip0->dst_address;
2156
2157             clib_memcpy (h0->link_layer_option.ethernet_address,
2158                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
2159
2160             /* $$$$ appears we need this; why is the checksum non-zero? */
2161             h0->neighbor.icmp.checksum = 0;
2162             h0->neighbor.icmp.checksum =
2163               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
2164                                                  &bogus_length);
2165
2166             ASSERT (bogus_length == 0);
2167
2168             vlib_buffer_copy_trace_flag (vm, p0, bi0);
2169             b0 = vlib_get_buffer (vm, bi0);
2170             vnet_buffer (b0)->sw_if_index[VLIB_TX]
2171               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
2172
2173             /* Add rewrite/encap string. */
2174             vnet_rewrite_one_header (adj0[0], h0,
2175                                      sizeof (ethernet_header_t));
2176             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2177
2178             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
2179
2180             vlib_set_next_frame_buffer (vm, node, next0, bi0);
2181           }
2182         }
2183
2184       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
2185                            n_left_to_next_drop);
2186     }
2187
2188   return frame->n_vectors;
2189 }
2190
2191 static char * ip6_discover_neighbor_error_strings[] = {
2192   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
2193   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
2194   = "neighbor solicitations sent",
2195   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
2196     = "no source address for ND solicitation",
2197 };
2198
2199 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
2200   .function = ip6_discover_neighbor,
2201   .name = "ip6-discover-neighbor",
2202   .vector_size = sizeof (u32),
2203
2204   .format_trace = format_ip6_forward_next_trace,
2205
2206   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
2207   .error_strings = ip6_discover_neighbor_error_strings,
2208
2209   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
2210   .next_nodes = {
2211     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
2212     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
2213   },
2214 };
2215
2216 clib_error_t *
2217 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
2218 {
2219   vnet_main_t * vnm = vnet_get_main();
2220   ip6_main_t * im = &ip6_main;
2221   icmp6_neighbor_solicitation_header_t * h;
2222   ip6_address_t * src;
2223   ip_interface_address_t * ia;
2224   ip_adjacency_t * adj;
2225   vnet_hw_interface_t * hi;
2226   vnet_sw_interface_t * si;
2227   vlib_buffer_t * b;
2228   u32 bi = 0;
2229   int bogus_length;
2230
2231   si = vnet_get_sw_interface (vnm, sw_if_index);
2232
2233   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
2234     {
2235       return clib_error_return (0, "%U: interface %U down",
2236                                 format_ip6_address, dst,
2237                                 format_vnet_sw_if_index_name, vnm,
2238                                 sw_if_index);
2239     }
2240
2241   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2242   if (! src)
2243     {
2244       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2245       return clib_error_return
2246         (0, "no matching interface address for destination %U (interface %U)",
2247          format_ip6_address, dst,
2248          format_vnet_sw_if_index_name, vnm, sw_if_index);
2249     }
2250
2251   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
2252
2253   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2254
2255   /* Destination address is a solicited node multicast address.  We need to fill in
2256      the low 24 bits with low 24 bits of target's address. */
2257   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
2258   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
2259   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
2260
2261   h->ip.src_address = src[0];
2262   h->neighbor.target_address = dst[0];
2263
2264   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
2265
2266   h->neighbor.icmp.checksum =
2267     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
2268   ASSERT(bogus_length == 0);
2269
2270   b = vlib_get_buffer (vm, bi);
2271   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2272
2273   /* Add encapsulation string for software interface (e.g. ethernet header). */
2274   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
2275   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2276   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2277
2278   {
2279     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
2280     u32 * to_next = vlib_frame_vector_args (f);
2281     to_next[0] = bi;
2282     f->n_vectors = 1;
2283     vlib_put_frame_to_node (vm, hi->output_node_index, f);
2284   }
2285
2286   return /* no error */ 0;
2287 }
2288
2289 typedef enum {
2290   IP6_REWRITE_NEXT_DROP,
2291   IP6_REWRITE_NEXT_ICMP_ERROR,
2292 } ip6_rewrite_next_t;
2293
2294 always_inline uword
2295 ip6_rewrite_inline (vlib_main_t * vm,
2296                     vlib_node_runtime_t * node,
2297                     vlib_frame_t * frame,
2298                     int rewrite_for_locally_received_packets)
2299 {
2300   ip_lookup_main_t * lm = &ip6_main.lookup_main;
2301   u32 * from = vlib_frame_vector_args (frame);
2302   u32 n_left_from, n_left_to_next, * to_next, next_index;
2303   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
2304   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
2305
2306   n_left_from = frame->n_vectors;
2307   next_index = node->cached_next_index;
2308   u32 cpu_index = os_get_cpu_number();
2309
2310   while (n_left_from > 0)
2311     {
2312       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2313
2314       while (n_left_from >= 4 && n_left_to_next >= 2)
2315         {
2316           ip_adjacency_t * adj0, * adj1;
2317           vlib_buffer_t * p0, * p1;
2318           ip6_header_t * ip0, * ip1;
2319           u32 pi0, rw_len0, next0, error0, adj_index0;
2320           u32 pi1, rw_len1, next1, error1, adj_index1;
2321
2322           /* Prefetch next iteration. */
2323           {
2324             vlib_buffer_t * p2, * p3;
2325
2326             p2 = vlib_get_buffer (vm, from[2]);
2327             p3 = vlib_get_buffer (vm, from[3]);
2328
2329             vlib_prefetch_buffer_header (p2, LOAD);
2330             vlib_prefetch_buffer_header (p3, LOAD);
2331
2332             CLIB_PREFETCH (p2->pre_data, 32, STORE);
2333             CLIB_PREFETCH (p3->pre_data, 32, STORE);
2334
2335             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2336             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2337           }
2338
2339           pi0 = to_next[0] = from[0];
2340           pi1 = to_next[1] = from[1];
2341
2342           from += 2;
2343           n_left_from -= 2;
2344           to_next += 2;
2345           n_left_to_next -= 2;
2346
2347           p0 = vlib_get_buffer (vm, pi0);
2348           p1 = vlib_get_buffer (vm, pi1);
2349
2350           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2351           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
2352
2353           /* We should never rewrite a pkt using the MISS adjacency */
2354           ASSERT(adj_index0 && adj_index1);
2355
2356           ip0 = vlib_buffer_get_current (p0);
2357           ip1 = vlib_buffer_get_current (p1);
2358
2359           error0 = error1 = IP6_ERROR_NONE;
2360           next0 = next1 = IP6_REWRITE_NEXT_DROP;
2361
2362           if (! rewrite_for_locally_received_packets)
2363             {
2364               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
2365
2366               /* Input node should have reject packets with hop limit 0. */
2367               ASSERT (ip0->hop_limit > 0);
2368               ASSERT (ip1->hop_limit > 0);
2369
2370               hop_limit0 -= 1;
2371               hop_limit1 -= 1;
2372
2373               ip0->hop_limit = hop_limit0;
2374               ip1->hop_limit = hop_limit1;
2375
2376               /*
2377                * If the hop count drops below 1 when forwarding, generate
2378                * an ICMP response.
2379                */
2380               if (PREDICT_FALSE(hop_limit0 <= 0))
2381                 {
2382                   error0 = IP6_ERROR_TIME_EXPIRED;
2383                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2384                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2385                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2386                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2387                 }
2388               if (PREDICT_FALSE(hop_limit1 <= 0))
2389                 {
2390                   error1 = IP6_ERROR_TIME_EXPIRED;
2391                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
2392                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2393                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
2394                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2395                 }
2396             }
2397
2398           adj0 = ip_get_adjacency (lm, adj_index0);
2399           adj1 = ip_get_adjacency (lm, adj_index1);
2400
2401           if (rewrite_for_locally_received_packets)
2402             {
2403               /*
2404                * If someone sends e.g. an icmp6 w/ src = dst = interface addr,
2405                * we end up here with a local adjacency in hand
2406                */
2407               if (PREDICT_FALSE(adj0->lookup_next_index
2408                                 == IP_LOOKUP_NEXT_LOCAL))
2409                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2410               if (PREDICT_FALSE(adj1->lookup_next_index
2411                                 == IP_LOOKUP_NEXT_LOCAL))
2412                 error1 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2413             }
2414
2415           rw_len0 = adj0[0].rewrite_header.data_bytes;
2416           rw_len1 = adj1[0].rewrite_header.data_bytes;
2417
2418           vlib_increment_combined_counter (&lm->adjacency_counters,
2419                                            cpu_index,
2420                                            adj_index0,
2421                                            /* packet increment */ 0,
2422                                            /* byte increment */ rw_len0);
2423           vlib_increment_combined_counter (&lm->adjacency_counters,
2424                                            cpu_index,
2425                                            adj_index1,
2426                                            /* packet increment */ 0,
2427                                            /* byte increment */ rw_len1);
2428
2429           /* Check MTU of outgoing interface. */
2430           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2431                     ? IP6_ERROR_MTU_EXCEEDED
2432                     : error0);
2433           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2434                     ? IP6_ERROR_MTU_EXCEEDED
2435                     : error1);
2436
2437           /* Don't adjust the buffer for hop count issue; icmp-error node
2438            * wants to see the IP headerr */
2439           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2440             {
2441               p0->current_data -= rw_len0;
2442               p0->current_length += rw_len0;
2443
2444               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2445                   adj0[0].rewrite_header.sw_if_index;
2446               next0 = adj0[0].rewrite_header.next_index;
2447             }
2448           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
2449             {
2450               p1->current_data -= rw_len1;
2451               p1->current_length += rw_len1;
2452
2453               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2454                   adj1[0].rewrite_header.sw_if_index;
2455               next1 = adj1[0].rewrite_header.next_index;
2456             }
2457
2458           /* Guess we are only writing on simple Ethernet header. */
2459           vnet_rewrite_two_headers (adj0[0], adj1[0],
2460                                     ip0, ip1,
2461                                     sizeof (ethernet_header_t));
2462
2463           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2464                                            to_next, n_left_to_next,
2465                                            pi0, pi1, next0, next1);
2466         }
2467
2468       while (n_left_from > 0 && n_left_to_next > 0)
2469         {
2470           ip_adjacency_t * adj0;
2471           vlib_buffer_t * p0;
2472           ip6_header_t * ip0;
2473           u32 pi0, rw_len0;
2474           u32 adj_index0, next0, error0;
2475
2476           pi0 = to_next[0] = from[0];
2477
2478           p0 = vlib_get_buffer (vm, pi0);
2479
2480           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2481
2482           /* We should never rewrite a pkt using the MISS adjacency */
2483           ASSERT(adj_index0);
2484
2485           adj0 = ip_get_adjacency (lm, adj_index0);
2486
2487           ip0 = vlib_buffer_get_current (p0);
2488
2489           error0 = IP6_ERROR_NONE;
2490           next0 = IP6_REWRITE_NEXT_DROP;
2491
2492           /* Check hop limit */
2493           if (! rewrite_for_locally_received_packets)
2494             {
2495               i32 hop_limit0 = ip0->hop_limit;
2496
2497               ASSERT (ip0->hop_limit > 0);
2498
2499               hop_limit0 -= 1;
2500
2501               ip0->hop_limit = hop_limit0;
2502
2503               if (PREDICT_FALSE(hop_limit0 <= 0))
2504                 {
2505                   /*
2506                    * If the hop count drops below 1 when forwarding, generate
2507                    * an ICMP response.
2508                    */
2509                   error0 = IP6_ERROR_TIME_EXPIRED;
2510                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2511                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2512                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2513                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2514                 }
2515             }
2516
2517           if (rewrite_for_locally_received_packets)
2518             {
2519               if (PREDICT_FALSE(adj0->lookup_next_index
2520                                 == IP_LOOKUP_NEXT_LOCAL))
2521                 error0 = IP6_ERROR_SPOOFED_LOCAL_PACKETS;
2522             }
2523
2524           /* Guess we are only writing on simple Ethernet header. */
2525           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2526
2527           /* Update packet buffer attributes/set output interface. */
2528           rw_len0 = adj0[0].rewrite_header.data_bytes;
2529
2530           vlib_increment_combined_counter (&lm->adjacency_counters,
2531                                            cpu_index,
2532                                            adj_index0,
2533                                            /* packet increment */ 0,
2534                                            /* byte increment */ rw_len0);
2535
2536           /* Check MTU of outgoing interface. */
2537           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2538                     ? IP6_ERROR_MTU_EXCEEDED
2539                     : error0);
2540
2541           /* Don't adjust the buffer for hop count issue; icmp-error node
2542            * wants to see the IP headerr */
2543           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2544             {
2545               p0->current_data -= rw_len0;
2546               p0->current_length += rw_len0;
2547
2548               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2549                   adj0[0].rewrite_header.sw_if_index;
2550               next0 = adj0[0].rewrite_header.next_index;
2551             }
2552
2553           p0->error = error_node->errors[error0];
2554
2555           from += 1;
2556           n_left_from -= 1;
2557           to_next += 1;
2558           n_left_to_next -= 1;
2559
2560           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2561                                            to_next, n_left_to_next,
2562                                            pi0, next0);
2563         }
2564
2565       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2566     }
2567
2568   /* Need to do trace after rewrites to pick up new packet data. */
2569   if (node->flags & VLIB_NODE_FLAG_TRACE)
2570     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2571
2572   return frame->n_vectors;
2573 }
2574
2575 static uword
2576 ip6_rewrite_transit (vlib_main_t * vm,
2577                      vlib_node_runtime_t * node,
2578                      vlib_frame_t * frame)
2579 {
2580   return ip6_rewrite_inline (vm, node, frame,
2581                              /* rewrite_for_locally_received_packets */ 0);
2582 }
2583
2584 static uword
2585 ip6_rewrite_local (vlib_main_t * vm,
2586                    vlib_node_runtime_t * node,
2587                    vlib_frame_t * frame)
2588 {
2589   return ip6_rewrite_inline (vm, node, frame,
2590                              /* rewrite_for_locally_received_packets */ 1);
2591 }
2592
2593 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2594   .function = ip6_rewrite_transit,
2595   .name = "ip6-rewrite",
2596   .vector_size = sizeof (u32),
2597
2598   .format_trace = format_ip6_rewrite_trace,
2599
2600   .n_next_nodes = 2,
2601   .next_nodes = {
2602     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2603     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2604   },
2605 };
2606
2607 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit)
2608
2609 VLIB_REGISTER_NODE (ip6_rewrite_local_node) = {
2610   .function = ip6_rewrite_local,
2611   .name = "ip6-rewrite-local",
2612   .vector_size = sizeof (u32),
2613
2614   .sibling_of = "ip6-rewrite",
2615
2616   .format_trace = format_ip6_rewrite_trace,
2617
2618   .n_next_nodes = 0,
2619 };
2620
2621 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local)
2622
2623 /*
2624  * Hop-by-Hop handling
2625  */
2626
2627 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2628
2629 #define foreach_ip6_hop_by_hop_error \
2630 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2631 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2632 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2633
2634 typedef enum {
2635 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2636   foreach_ip6_hop_by_hop_error
2637 #undef _
2638   IP6_HOP_BY_HOP_N_ERROR,
2639 } ip6_hop_by_hop_error_t;
2640
2641 /*
2642  * Primary h-b-h handler trace support
2643  * We work pretty hard on the problem for obvious reasons
2644  */
2645 typedef struct {
2646   u32 next_index;
2647   u32 trace_len;
2648   u8 option_data[256];
2649 } ip6_hop_by_hop_trace_t;
2650
2651 vlib_node_registration_t ip6_hop_by_hop_node;
2652
2653 static char * ip6_hop_by_hop_error_strings[] = {
2654 #define _(sym,string) string,
2655   foreach_ip6_hop_by_hop_error
2656 #undef _
2657 };
2658
2659 static u8 *
2660 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2661 {
2662   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2663   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2664   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2665   ip6_hop_by_hop_header_t *hbh0;
2666   ip6_hop_by_hop_option_t *opt0, *limit0;
2667   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2668
2669   u8 type0;
2670
2671   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2672
2673   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2674               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2675
2676   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2677   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2678
2679   while (opt0 < limit0) {
2680     type0 = opt0->type;
2681     switch (type0) {
2682     case 0: /* Pad, just stop */
2683       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2684       break;
2685
2686     default:
2687       if (hm->trace[type0]) {
2688         s = (*hm->trace[type0])(s, opt0);
2689       } else {
2690         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2691       }
2692       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2693       break;
2694     }
2695   }
2696   return s;
2697 }
2698
2699 /*
2700  * Process the Hop-by-Hop Options header
2701  */
2702 static uword
2703 ip6_hop_by_hop (vlib_main_t * vm,
2704                 vlib_node_runtime_t * node,
2705                 vlib_frame_t * frame)
2706 {
2707   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2708   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2709   u32 n_left_from, *from, *to_next;
2710   ip_lookup_next_t next_index;
2711   ip6_main_t * im = &ip6_main;
2712   ip_lookup_main_t *lm = &im->lookup_main;
2713
2714   from = vlib_frame_vector_args (frame);
2715   n_left_from = frame->n_vectors;
2716   next_index = node->cached_next_index;
2717
2718   while (n_left_from > 0) {
2719     u32 n_left_to_next;
2720
2721     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2722
2723     while (n_left_from > 0 && n_left_to_next > 0) {
2724       u32 bi0;
2725       vlib_buffer_t * b0;
2726       u32 next0;
2727       ip6_header_t * ip0;
2728       ip6_hop_by_hop_header_t *hbh0;
2729       ip6_hop_by_hop_option_t *opt0, *limit0;
2730       u8 type0;
2731       u8 error0 = 0;
2732
2733       /* Speculatively enqueue b0 to the current next frame */
2734       bi0 = from[0];
2735       to_next[0] = bi0;
2736       from += 1;
2737       to_next += 1;
2738       n_left_from -= 1;
2739       n_left_to_next -= 1;
2740
2741       b0 = vlib_get_buffer (vm, bi0);
2742       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2743       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2744       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2745       next0 = adj0->lookup_next_index;
2746
2747       ip0 = vlib_buffer_get_current (b0);
2748       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2749       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2750       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2751
2752       /*
2753        * Basic validity checks
2754        */
2755       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2756         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2757         next0 = IP_LOOKUP_NEXT_DROP;
2758         goto out0;
2759       }
2760
2761       /* Scan the set of h-b-h options, process ones that we understand */
2762       while (opt0 < limit0) {
2763         type0 = opt0->type;
2764         switch (type0) {
2765         case 0: /* Pad1 */
2766           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2767           continue;
2768         case 1: /* PadN */
2769           break;
2770         default:
2771           if (hm->options[type0]) {
2772             if ((*hm->options[type0])(b0, ip0, opt0) < 0) {
2773               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2774               goto out0;
2775             }
2776           } else {
2777             /* Unrecognized mandatory option, check the two high order bits */
2778             switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS) {
2779             case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2780               break;
2781             case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2782               next0 = IP_LOOKUP_NEXT_DROP;
2783               break;
2784             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2785               next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2786               icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2787                                           ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2788               break;
2789             case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2790               if (!ip6_address_is_multicast(&ip0->dst_address)) {
2791                 next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2792                 icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2793                                             ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2794               } else {
2795                 next0 =  IP_LOOKUP_NEXT_DROP;
2796               }
2797               break;
2798             }
2799             error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2800             goto out0;
2801           }
2802         }
2803         opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2804       }
2805
2806     out0:
2807       /* Has the classifier flagged this buffer for special treatment? */
2808       if ((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index == OI_DECAP))
2809         next0 = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2810
2811       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2812         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2813         u32 trace_len = (hbh0->length + 1) << 3;
2814         t->next_index = next0;
2815         /* Capture the h-b-h option verbatim */
2816         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2817         t->trace_len = trace_len;
2818         clib_memcpy(t->option_data, hbh0, trace_len);
2819       }
2820
2821       b0->error = error_node->errors[error0];
2822
2823       /* verify speculative enqueue, maybe switch current next frame */
2824       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2825     }
2826     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2827   }
2828   return frame->n_vectors;
2829 }
2830
2831 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2832   .function = ip6_hop_by_hop,
2833   .name = "ip6-hop-by-hop",
2834   .sibling_of = "ip6-lookup",
2835   .vector_size = sizeof (u32),
2836   .format_trace = format_ip6_hop_by_hop_trace,
2837   .type = VLIB_NODE_TYPE_INTERNAL,
2838   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2839   .error_strings = ip6_hop_by_hop_error_strings,
2840   .n_next_nodes = 0,
2841 };
2842
2843 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop)
2844
2845 static clib_error_t *
2846 ip6_hop_by_hop_init (vlib_main_t * vm)
2847 {
2848   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2849   memset(hm->options, 0, sizeof(hm->options));
2850   memset(hm->trace, 0, sizeof(hm->trace));
2851
2852   return (0);
2853 }
2854
2855 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2856
2857 int
2858 ip6_hbh_register_option (u8 option,
2859                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2860                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2861 {
2862   ip6_main_t * im = &ip6_main;
2863   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2864
2865   ASSERT (option < ARRAY_LEN (hm->options));
2866
2867   /* Already registered */
2868   if (hm->options[option])
2869     return (-1);
2870
2871   hm->options[option] = options;
2872   hm->trace[option] = trace;
2873
2874   /* Set global variable */
2875   im->hbh_enabled = 1;
2876
2877   return (0);
2878 }
2879
2880 int
2881 ip6_hbh_unregister_option (u8 option)
2882 {
2883   ip6_main_t * im = &ip6_main;
2884   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2885
2886   ASSERT (option < ARRAY_LEN (hm->options));
2887
2888   /* Not registered */
2889   if (!hm->options[option])
2890     return (-1);
2891
2892   hm->options[option] = NULL;
2893   hm->trace[option] = NULL;
2894
2895   /* Disable global knob if this was the last option configured */
2896   int i;
2897   bool found = false;
2898   for (i = 0; i < 256; i++) {
2899     if (hm->options[option]) {
2900       found = true;
2901       break;
2902     }
2903   }
2904   if (!found)
2905     im->hbh_enabled = 0;
2906
2907   return (0);
2908 }
2909
2910 /* Global IP6 main. */
2911 ip6_main_t ip6_main;
2912
2913 static clib_error_t *
2914 ip6_lookup_init (vlib_main_t * vm)
2915 {
2916   ip6_main_t * im = &ip6_main;
2917   clib_error_t * error;
2918   uword i;
2919
2920   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2921     {
2922       u32 j, i0, i1;
2923
2924       i0 = i / 32;
2925       i1 = i % 32;
2926
2927       for (j = 0; j < i0; j++)
2928         im->fib_masks[i].as_u32[j] = ~0;
2929
2930       if (i1)
2931         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2932     }
2933
2934   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2935
2936   if (im->lookup_table_nbuckets == 0)
2937     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2938
2939   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2940
2941   if (im->lookup_table_size == 0)
2942     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2943
2944   BV(clib_bihash_init) (&im->ip6_lookup_table, "ip6 lookup table",
2945                         im->lookup_table_nbuckets,
2946                         im->lookup_table_size);
2947
2948   /* Create FIB with index 0 and table id of 0. */
2949   find_ip6_fib_by_table_index_or_id (im, /* table id */ 0, IP6_ROUTE_FLAG_TABLE_ID);
2950
2951   {
2952     pg_node_t * pn;
2953     pn = pg_get_node (ip6_lookup_node.index);
2954     pn->unformat_edit = unformat_pg_ip6_header;
2955   }
2956
2957   /* Unless explicitly configured, don't process HBH options */
2958   im->hbh_enabled = 0;
2959
2960   {
2961     icmp6_neighbor_solicitation_header_t p;
2962
2963     memset (&p, 0, sizeof (p));
2964
2965     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2966     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2967                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2968     p.ip.protocol = IP_PROTOCOL_ICMP6;
2969     p.ip.hop_limit = 255;
2970     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2971
2972     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2973
2974     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2975     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2976
2977     vlib_packet_template_init (vm,
2978                                &im->discover_neighbor_packet_template,
2979                                &p, sizeof (p),
2980                                /* alloc chunk size */ 8,
2981                                "ip6 neighbor discovery");
2982   }
2983
2984   error = ip6_feature_init (vm, im);
2985
2986   return error;
2987 }
2988
2989 VLIB_INIT_FUNCTION (ip6_lookup_init);
2990
2991 static clib_error_t *
2992 add_del_ip6_interface_table (vlib_main_t * vm,
2993                              unformat_input_t * input,
2994                              vlib_cli_command_t * cmd)
2995 {
2996   vnet_main_t * vnm = vnet_get_main();
2997   clib_error_t * error = 0;
2998   u32 sw_if_index, table_id;
2999
3000   sw_if_index = ~0;
3001
3002   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
3003     {
3004       error = clib_error_return (0, "unknown interface `%U'",
3005                                  format_unformat_error, input);
3006       goto done;
3007     }
3008
3009   if (unformat (input, "%d", &table_id))
3010     ;
3011   else
3012     {
3013       error = clib_error_return (0, "expected table id `%U'",
3014                                  format_unformat_error, input);
3015       goto done;
3016     }
3017
3018   {
3019     ip6_main_t * im = &ip6_main;
3020     ip6_fib_t * fib =
3021       find_ip6_fib_by_table_index_or_id (im, table_id, IP6_ROUTE_FLAG_TABLE_ID);
3022
3023     if (fib)
3024       {
3025         vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
3026         im->fib_index_by_sw_if_index[sw_if_index] = fib->index;
3027     }
3028   }
3029
3030  done:
3031   return error;
3032 }
3033
3034 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) = {
3035   .path = "set interface ip6 table",
3036   .function = add_del_ip6_interface_table,
3037   .short_help = "set interface ip6 table <intfc> <table-id>"
3038 };
3039
3040 void
3041 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
3042                                                   u8 *mac)
3043 {
3044   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
3045   /* Invert the "u" bit */
3046   ip->as_u8 [8] = mac[0] ^ (1<<1);
3047   ip->as_u8 [9] = mac[1];
3048   ip->as_u8 [10] = mac[2];
3049   ip->as_u8 [11] = 0xFF;
3050   ip->as_u8 [12] = 0xFE;
3051   ip->as_u8 [13] = mac[3];
3052   ip->as_u8 [14] = mac[4];
3053   ip->as_u8 [15] = mac[5];
3054 }
3055
3056 void
3057 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
3058                                                   ip6_address_t *ip)
3059 {
3060   /* Invert the previously inverted "u" bit */
3061   mac[0] = ip->as_u8 [8] ^ (1<<1);
3062   mac[1] = ip->as_u8 [9];
3063   mac[2] = ip->as_u8 [10];
3064   mac[3] = ip->as_u8 [13];
3065   mac[4] = ip->as_u8 [14];
3066   mac[5] = ip->as_u8 [15];
3067 }
3068
3069 static clib_error_t *
3070 test_ip6_link_command_fn (vlib_main_t * vm,
3071                           unformat_input_t * input,
3072                           vlib_cli_command_t * cmd)
3073 {
3074   u8 mac[6];
3075   ip6_address_t _a, *a = &_a;
3076
3077   if (unformat (input, "%U", unformat_ethernet_address, mac))
3078     {
3079       ip6_link_local_address_from_ethernet_mac_address (a, mac);
3080       vlib_cli_output (vm, "Link local address: %U",
3081                        format_ip6_address, a);
3082       ip6_ethernet_mac_address_from_link_local_address (mac, a);
3083       vlib_cli_output (vm, "Original MAC address: %U",
3084                        format_ethernet_address, mac);
3085     }
3086
3087   return 0;
3088 }
3089
3090 VLIB_CLI_COMMAND (test_link_command, static) = {
3091   .path = "test ip6 link",
3092   .function = test_ip6_link_command_fn,
3093   .short_help = "test ip6 link <mac-address>",
3094 };
3095
3096 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
3097 {
3098   ip6_main_t * im6 = &ip6_main;
3099   ip6_fib_t * fib;
3100   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
3101
3102   if (p == 0)
3103     return -1;
3104
3105   fib = vec_elt_at_index (im6->fibs, p[0]);
3106
3107   fib->flow_hash_config = flow_hash_config;
3108   return 1;
3109 }
3110
3111 static clib_error_t *
3112 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
3113                               unformat_input_t * input,
3114                               vlib_cli_command_t * cmd)
3115 {
3116   int matched = 0;
3117   u32 table_id = 0;
3118   u32 flow_hash_config = 0;
3119   int rv;
3120
3121   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3122     if (unformat (input, "table %d", &table_id))
3123       matched = 1;
3124 #define _(a,v) \
3125     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3126     foreach_flow_hash_bit
3127 #undef _
3128     else break;
3129   }
3130
3131   if (matched == 0)
3132     return clib_error_return (0, "unknown input `%U'",
3133                               format_unformat_error, input);
3134
3135   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
3136   switch (rv)
3137     {
3138     case 1:
3139       break;
3140
3141     case -1:
3142       return clib_error_return (0, "no such FIB table %d", table_id);
3143
3144     default:
3145       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3146       break;
3147     }
3148
3149   return 0;
3150 }
3151
3152 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
3153     .path = "set ip6 flow-hash",
3154     .short_help =
3155     "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
3156     .function = set_ip6_flow_hash_command_fn,
3157 };
3158
3159 static clib_error_t *
3160 show_ip6_local_command_fn (vlib_main_t * vm,
3161                            unformat_input_t * input,
3162                            vlib_cli_command_t * cmd)
3163 {
3164   ip6_main_t * im = &ip6_main;
3165   ip_lookup_main_t * lm = &im->lookup_main;
3166   int i;
3167
3168   vlib_cli_output (vm, "Protocols handled by ip6_local");
3169   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
3170     {
3171       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3172         vlib_cli_output (vm, "%d", i);
3173     }
3174   return 0;
3175 }
3176
3177
3178
3179 VLIB_CLI_COMMAND (show_ip_local, static) = {
3180   .path = "show ip6 local",
3181   .function = show_ip6_local_command_fn,
3182   .short_help = "Show ip6 local protocol table",
3183 };
3184
3185 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3186                                  u32 table_index)
3187 {
3188   vnet_main_t * vnm = vnet_get_main();
3189   vnet_interface_main_t * im = &vnm->interface_main;
3190   ip6_main_t * ipm = &ip6_main;
3191   ip_lookup_main_t * lm = &ipm->lookup_main;
3192   vnet_classify_main_t * cm = &vnet_classify_main;
3193
3194   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3195     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3196
3197   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3198     return VNET_API_ERROR_NO_SUCH_ENTRY;
3199
3200   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3201   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3202
3203   return 0;
3204 }
3205
3206 static clib_error_t *
3207 set_ip6_classify_command_fn (vlib_main_t * vm,
3208                              unformat_input_t * input,
3209                              vlib_cli_command_t * cmd)
3210 {
3211   u32 table_index = ~0;
3212   int table_index_set = 0;
3213   u32 sw_if_index = ~0;
3214   int rv;
3215
3216   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3217     if (unformat (input, "table-index %d", &table_index))
3218       table_index_set = 1;
3219     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3220                        vnet_get_main(), &sw_if_index))
3221         ;
3222     else
3223         break;
3224   }
3225
3226   if (table_index_set == 0)
3227       return clib_error_return (0, "classify table-index must be specified");
3228
3229   if (sw_if_index == ~0)
3230     return clib_error_return (0, "interface / subif must be specified");
3231
3232   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3233
3234   switch (rv)
3235     {
3236     case 0:
3237       break;
3238
3239     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3240       return clib_error_return (0, "No such interface");
3241
3242     case VNET_API_ERROR_NO_SUCH_ENTRY:
3243       return clib_error_return (0, "No such classifier table");
3244     }
3245   return 0;
3246 }
3247
3248 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3249     .path = "set ip6 classify",
3250     .short_help =
3251     "set ip6 classify intfc <int> table-index <index>",
3252     .function = set_ip6_classify_command_fn,
3253 };
3254
3255 static clib_error_t *
3256 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3257 {
3258   ip6_main_t * im = &ip6_main;
3259   uword heapsize = 0;
3260   u32 tmp;
3261   u32 nbuckets = 0;
3262
3263   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3264     if (unformat (input, "hash-buckets %d", &tmp))
3265       nbuckets = tmp;
3266     else if (unformat (input, "heap-size %dm", &tmp))
3267       heapsize = ((u64)tmp) << 20;
3268     else if (unformat (input, "heap-size %dM", &tmp))
3269       heapsize = ((u64)tmp) << 20;
3270     else if (unformat (input, "heap-size %dg", &tmp))
3271       heapsize = ((u64)tmp) << 30;
3272     else if (unformat (input, "heap-size %dG", &tmp))
3273       heapsize = ((u64)tmp) << 30;
3274     else
3275       return clib_error_return (0, "unknown input '%U'",
3276                                 format_unformat_error, input);
3277   }
3278
3279   im->lookup_table_nbuckets = nbuckets;
3280   im->lookup_table_size = heapsize;
3281
3282   return 0;
3283 }
3284
3285 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3286