src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_neighbor.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52
  53 #include <vppinfra/bihash_template.c>
  54 #include <vnet/ip/ip6_forward.h>
  55
  56 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  57 #define OI_DECAP   0x80000000
  58
  59 static void
  60 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
  61                           ip6_main_t * im, u32 fib_index,
  62                           ip_interface_address_t * a)
  63 {
  64   ip_lookup_main_t *lm = &im->lookup_main;
  65   ip6_address_t *address = ip_interface_address_get_address (lm, a);
  66   fib_prefix_t pfx = {
  67     .fp_len = a->address_length,
  68     .fp_proto = FIB_PROTOCOL_IP6,
  69     .fp_addr.ip6 = *address,
  70   };
  71
  72   if (a->address_length < 128)
  73     {
  74       fib_table_entry_update_one_path (fib_index,
  75                                        &pfx,
  76                                        FIB_SOURCE_INTERFACE,
  77                                        (FIB_ENTRY_FLAG_CONNECTED |
  78                                         FIB_ENTRY_FLAG_ATTACHED),
  79                                        DPO_PROTO_IP6,
  80                                        /* No next-hop address */
  81                                        NULL, sw_if_index,
  82                                        /* invalid FIB index */
  83                                        ~0, 1,
  84                                        /* no label stack */
  85                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
  86     }
  87
  88   pfx.fp_len = 128;
  89   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
  90     {
  91       u32 classify_table_index =
  92         lm->classify_table_index_by_sw_if_index[sw_if_index];
  93       if (classify_table_index != (u32) ~ 0)
  94         {
  95           dpo_id_t dpo = DPO_INVALID;
  96
  97           dpo_set (&dpo,
  98                    DPO_CLASSIFY,
  99                    DPO_PROTO_IP6,
 100                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 101
 102           fib_table_entry_special_dpo_add (fib_index,
 103                                            &pfx,
 104                                            FIB_SOURCE_CLASSIFY,
 105                                            FIB_ENTRY_FLAG_NONE, &dpo);
 106           dpo_reset (&dpo);
 107         }
 108     }
 109
 110   fib_table_entry_update_one_path (fib_index, &pfx,
 111                                    FIB_SOURCE_INTERFACE,
 112                                    (FIB_ENTRY_FLAG_CONNECTED |
 113                                     FIB_ENTRY_FLAG_LOCAL),
 114                                    DPO_PROTO_IP6,
 115                                    &pfx.fp_addr,
 116                                    sw_if_index, ~0,
 117                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 118 }
 119
 120 static void
 121 ip6_del_interface_routes (ip6_main_t * im,
 122                           u32 fib_index,
 123                           ip6_address_t * address, u32 address_length)
 124 {
 125   fib_prefix_t pfx = {
 126     .fp_len = address_length,
 127     .fp_proto = FIB_PROTOCOL_IP6,
 128     .fp_addr.ip6 = *address,
 129   };
 130
 131   if (pfx.fp_len < 128)
 132     {
 133       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 134
 135     }
 136
 137   pfx.fp_len = 128;
 138   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 139 }
 140
 141 void
 142 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 143 {
 144   ip6_main_t *im = &ip6_main;
 145
 146   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 147
 148   /*
 149    * enable/disable only on the 1<->0 transition
 150    */
 151   if (is_enable)
 152     {
 153       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 154         return;
 155     }
 156   else
 157     {
 158       /* The ref count is 0 when an address is removed from an interface that has
 159        * no address - this is not a ciritical error */
 160       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 161           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 162         return;
 163     }
 164
 165   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 166                                !is_enable, 0, 0);
 167
 168   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 169                                sw_if_index, !is_enable, 0, 0);
 170 }
 171
 172 /* get first interface address */
 173 ip6_address_t *
 174 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 175 {
 176   ip_lookup_main_t *lm = &im->lookup_main;
 177   ip_interface_address_t *ia = 0;
 178   ip6_address_t *result = 0;
 179
 180   /* *INDENT-OFF* */
 181   foreach_ip_interface_address (lm, ia, sw_if_index,
 182                                 1 /* honor unnumbered */,
 183   ({
 184     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 185     result = a;
 186     break;
 187   }));
 188   /* *INDENT-ON* */
 189   return result;
 190 }
 191
 192 clib_error_t *
 193 ip6_add_del_interface_address (vlib_main_t * vm,
 194                                u32 sw_if_index,
 195                                ip6_address_t * address,
 196                                u32 address_length, u32 is_del)
 197 {
 198   vnet_main_t *vnm = vnet_get_main ();
 199   ip6_main_t *im = &ip6_main;
 200   ip_lookup_main_t *lm = &im->lookup_main;
 201   clib_error_t *error;
 202   u32 if_address_index;
 203   ip6_address_fib_t ip6_af, *addr_fib = 0;
 204   ip6_address_t ll_addr;
 205
 206   /* local0 interface doesn't support IP addressing */
 207   if (sw_if_index == 0)
 208     {
 209       return
 210         clib_error_create ("local0 interface doesn't support IP addressing");
 211     }
 212
 213   if (ip6_address_is_link_local_unicast (address))
 214     {
 215       if (address_length != 128)
 216         {
 217           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 218           return
 219             clib_error_create
 220             ("prefix length of link-local address must be 128");
 221         }
 222       if (!is_del)
 223         {
 224           return ip6_neighbor_set_link_local_address (vm, sw_if_index,
 225                                                       address);
 226         }
 227       else
 228         {
 229           ll_addr = ip6_neighbor_get_link_local_address (sw_if_index);
 230           if (ip6_address_is_equal (&ll_addr, address))
 231             {
 232               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 233               return clib_error_create ("address not deletable");
 234             }
 235           else
 236             {
 237               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 238               return clib_error_create ("address not found");
 239             }
 240         }
 241     }
 242
 243   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 244   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 245
 246   ip6_addr_fib_init (&ip6_af, address,
 247                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 248   vec_add1 (addr_fib, ip6_af);
 249
 250   /* *INDENT-OFF* */
 251   if (!is_del)
 252     {
 253       /* When adding an address check that it does not conflict
 254          with an existing address on any interface in this table. */
 255       ip_interface_address_t *ia;
 256       vnet_sw_interface_t *sif;
 257
 258       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 259       ({
 260           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 261               im->fib_index_by_sw_if_index[sif->sw_if_index])
 262             {
 263               foreach_ip_interface_address
 264                 (&im->lookup_main, ia, sif->sw_if_index,
 265                  0 /* honor unnumbered */ ,
 266                  ({
 267                    ip6_address_t * x =
 268                      ip_interface_address_get_address
 269                      (&im->lookup_main, ia);
 270                    if (ip6_destination_matches_route
 271                        (im, address, x, ia->address_length) ||
 272                        ip6_destination_matches_route (im,
 273                                                       x,
 274                                                       address,
 275                                                       address_length))
 276                      {
 277                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 278                        return
 279                          clib_error_create
 280                          ("failed to add %U which conflicts with %U for interface %U",
 281                           format_ip6_address_and_length, address,
 282                           address_length,
 283                           format_ip6_address_and_length, x,
 284                           ia->address_length,
 285                           format_vnet_sw_if_index_name, vnm,
 286                           sif->sw_if_index);
 287                      }
 288                  }));
 289             }
 290       }));
 291     }
 292   /* *INDENT-ON* */
 293
 294   {
 295     uword elts_before = pool_elts (lm->if_address_pool);
 296
 297     error = ip_interface_address_add_del
 298       (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 299     if (error)
 300       goto done;
 301
 302     /* Pool did not grow: add duplicate address. */
 303     if (elts_before == pool_elts (lm->if_address_pool))
 304       goto done;
 305   }
 306
 307   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 308
 309   if (is_del)
 310     ip6_del_interface_routes (im, ip6_af.fib_index, address, address_length);
 311   else
 312     ip6_add_interface_routes (vnm, sw_if_index,
 313                               im, ip6_af.fib_index,
 314                               pool_elt_at_index (lm->if_address_pool,
 315                                                  if_address_index));
 316
 317   {
 318     ip6_add_del_interface_address_callback_t *cb;
 319     vec_foreach (cb, im->add_del_interface_address_callbacks)
 320       cb->function (im, cb->function_opaque, sw_if_index,
 321                     address, address_length, if_address_index, is_del);
 322   }
 323
 324 done:
 325   vec_free (addr_fib);
 326   return error;
 327 }
 328
 329 clib_error_t *
 330 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 331 {
 332   ip6_main_t *im = &ip6_main;
 333   ip_interface_address_t *ia;
 334   ip6_address_t *a;
 335   u32 is_admin_up, fib_index;
 336
 337   /* Fill in lookup tables with default table (0). */
 338   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 339
 340   vec_validate_init_empty (im->
 341                            lookup_main.if_address_pool_index_by_sw_if_index,
 342                            sw_if_index, ~0);
 343
 344   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 345
 346   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 347
 348   /* *INDENT-OFF* */
 349   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 350                                 0 /* honor unnumbered */,
 351   ({
 352     a = ip_interface_address_get_address (&im->lookup_main, ia);
 353     if (is_admin_up)
 354       ip6_add_interface_routes (vnm, sw_if_index,
 355                                 im, fib_index,
 356                                 ia);
 357     else
 358       ip6_del_interface_routes (im, fib_index,
 359                                 a, ia->address_length);
 360   }));
 361   /* *INDENT-ON* */
 362
 363   return 0;
 364 }
 365
 366 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 367
 368 /* Built-in ip6 unicast rx feature path definition */
 369 /* *INDENT-OFF* */
 370 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 371 {
 372   .arc_name  = "ip6-unicast",
 373   .start_nodes = VNET_FEATURES ("ip6-input"),
 374   .last_in_arc = "ip6-lookup",
 375   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 376 };
 377
 378 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 379 {
 380   .arc_name = "ip6-unicast",
 381   .node_name = "ip6-flow-classify",
 382   .runs_before = VNET_FEATURES ("ip6-inacl"),
 383 };
 384
 385 VNET_FEATURE_INIT (ip6_inacl, static) =
 386 {
 387   .arc_name = "ip6-unicast",
 388   .node_name = "ip6-inacl",
 389   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 390 };
 391
 392 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 393 {
 394   .arc_name = "ip6-unicast",
 395   .node_name = "ip6-policer-classify",
 396   .runs_before = VNET_FEATURES ("ipsec6-input"),
 397 };
 398
 399 VNET_FEATURE_INIT (ip6_ipsec, static) =
 400 {
 401   .arc_name = "ip6-unicast",
 402   .node_name = "ipsec6-input",
 403   .runs_before = VNET_FEATURES ("l2tp-decap"),
 404 };
 405
 406 VNET_FEATURE_INIT (ip6_l2tp, static) =
 407 {
 408   .arc_name = "ip6-unicast",
 409   .node_name = "l2tp-decap",
 410   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 411 };
 412
 413 VNET_FEATURE_INIT (ip6_vpath, static) =
 414 {
 415   .arc_name = "ip6-unicast",
 416   .node_name = "vpath-input-ip6",
 417   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 418 };
 419
 420 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 421 {
 422   .arc_name = "ip6-unicast",
 423   .node_name = "ip6-vxlan-bypass",
 424   .runs_before = VNET_FEATURES ("ip6-lookup"),
 425 };
 426
 427 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 428 {
 429   .arc_name = "ip6-unicast",
 430   .node_name = "ip6-not-enabled",
 431   .runs_before = VNET_FEATURES ("ip6-lookup"),
 432 };
 433
 434 VNET_FEATURE_INIT (ip6_lookup, static) =
 435 {
 436   .arc_name = "ip6-unicast",
 437   .node_name = "ip6-lookup",
 438   .runs_before = 0,  /*last feature*/
 439 };
 440
 441 /* Built-in ip6 multicast rx feature path definition (none now) */
 442 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 443 {
 444   .arc_name  = "ip6-multicast",
 445   .start_nodes = VNET_FEATURES ("ip6-input"),
 446   .last_in_arc = "ip6-mfib-forward-lookup",
 447   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 448 };
 449
 450 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 451   .arc_name = "ip6-multicast",
 452   .node_name = "vpath-input-ip6",
 453   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 454 };
 455
 456 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 457   .arc_name = "ip6-multicast",
 458   .node_name = "ip6-not-enabled",
 459   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 460 };
 461
 462 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 463   .arc_name = "ip6-multicast",
 464   .node_name = "ip6-mfib-forward-lookup",
 465   .runs_before = 0, /* last feature */
 466 };
 467
 468 /* Built-in ip4 tx feature path definition */
 469 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 470 {
 471   .arc_name  = "ip6-output",
 472   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 473   .last_in_arc = "interface-output",
 474   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 475 };
 476
 477 VNET_FEATURE_INIT (ip6_outacl, static) = {
 478   .arc_name = "ip6-output",
 479   .node_name = "ip6-outacl",
 480   .runs_before = VNET_FEATURES ("ipsec6-output"),
 481 };
 482
 483 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 484   .arc_name = "ip6-output",
 485   .node_name = "ipsec6-output",
 486   .runs_before = VNET_FEATURES ("interface-output"),
 487 };
 488
 489 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 490   .arc_name = "ip6-output",
 491   .node_name = "interface-output",
 492   .runs_before = 0, /* not before any other features */
 493 };
 494 /* *INDENT-ON* */
 495
 496 clib_error_t *
 497 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 498 {
 499   ip6_main_t *im = &ip6_main;
 500
 501   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 502   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 503
 504   if (!is_add)
 505     {
 506       /* Ensure that IPv6 is disabled */
 507       ip6_main_t *im6 = &ip6_main;
 508       ip_lookup_main_t *lm6 = &im6->lookup_main;
 509       ip_interface_address_t *ia = 0;
 510       ip6_address_t *address;
 511       vlib_main_t *vm = vlib_get_main ();
 512
 513       ip6_neighbor_sw_interface_add_del (vnm, sw_if_index, 0 /* is_add */ );
 514       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 515       /* *INDENT-OFF* */
 516       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 517       ({
 518         address = ip_interface_address_get_address (lm6, ia);
 519         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 520       }));
 521       /* *INDENT-ON* */
 522       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 523     }
 524
 525   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 526                                is_add, 0, 0);
 527
 528   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 529                                sw_if_index, is_add, 0, 0);
 530
 531   return /* no error */ 0;
 532 }
 533
 534 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 535
 536 static uword
 537 ip6_lookup (vlib_main_t * vm,
 538             vlib_node_runtime_t * node, vlib_frame_t * frame)
 539 {
 540   return ip6_lookup_inline (vm, node, frame);
 541 }
 542
 543 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 544
 545 /* *INDENT-OFF* */
 546 VLIB_REGISTER_NODE (ip6_lookup_node) =
 547 {
 548   .function = ip6_lookup,
 549   .name = "ip6-lookup",
 550   .vector_size = sizeof (u32),
 551   .format_trace = format_ip6_lookup_trace,
 552   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 553   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 554 };
 555 /* *INDENT-ON* */
 556
 557 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup);
 558
 559 static uword
 560 ip6_load_balance (vlib_main_t * vm,
 561                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 562 {
 563   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 564   u32 n_left_from, n_left_to_next, *from, *to_next;
 565   ip_lookup_next_t next;
 566   u32 thread_index = vm->thread_index;
 567   ip6_main_t *im = &ip6_main;
 568
 569   from = vlib_frame_vector_args (frame);
 570   n_left_from = frame->n_vectors;
 571   next = node->cached_next_index;
 572
 573   while (n_left_from > 0)
 574     {
 575       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 576
 577
 578       while (n_left_from >= 4 && n_left_to_next >= 2)
 579         {
 580           ip_lookup_next_t next0, next1;
 581           const load_balance_t *lb0, *lb1;
 582           vlib_buffer_t *p0, *p1;
 583           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 584           const ip6_header_t *ip0, *ip1;
 585           const dpo_id_t *dpo0, *dpo1;
 586
 587           /* Prefetch next iteration. */
 588           {
 589             vlib_buffer_t *p2, *p3;
 590
 591             p2 = vlib_get_buffer (vm, from[2]);
 592             p3 = vlib_get_buffer (vm, from[3]);
 593
 594             vlib_prefetch_buffer_header (p2, STORE);
 595             vlib_prefetch_buffer_header (p3, STORE);
 596
 597             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 598             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 599           }
 600
 601           pi0 = to_next[0] = from[0];
 602           pi1 = to_next[1] = from[1];
 603
 604           from += 2;
 605           n_left_from -= 2;
 606           to_next += 2;
 607           n_left_to_next -= 2;
 608
 609           p0 = vlib_get_buffer (vm, pi0);
 610           p1 = vlib_get_buffer (vm, pi1);
 611
 612           ip0 = vlib_buffer_get_current (p0);
 613           ip1 = vlib_buffer_get_current (p1);
 614           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 615           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 616
 617           lb0 = load_balance_get (lbi0);
 618           lb1 = load_balance_get (lbi1);
 619
 620           /*
 621            * this node is for via FIBs we can re-use the hash value from the
 622            * to node if present.
 623            * We don't want to use the same hash value at each level in the recursion
 624            * graph as that would lead to polarisation
 625            */
 626           hc0 = hc1 = 0;
 627
 628           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 629             {
 630               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 631                 {
 632                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 633                     vnet_buffer (p0)->ip.flow_hash >> 1;
 634                 }
 635               else
 636                 {
 637                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 638                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 639                 }
 640               dpo0 =
 641                 load_balance_get_fwd_bucket (lb0,
 642                                              (hc0 &
 643                                               lb0->lb_n_buckets_minus_1));
 644             }
 645           else
 646             {
 647               dpo0 = load_balance_get_bucket_i (lb0, 0);
 648             }
 649           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 650             {
 651               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 652                 {
 653                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 654                     vnet_buffer (p1)->ip.flow_hash >> 1;
 655                 }
 656               else
 657                 {
 658                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 659                     ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 660                 }
 661               dpo1 =
 662                 load_balance_get_fwd_bucket (lb1,
 663                                              (hc1 &
 664                                               lb1->lb_n_buckets_minus_1));
 665             }
 666           else
 667             {
 668               dpo1 = load_balance_get_bucket_i (lb1, 0);
 669             }
 670
 671           next0 = dpo0->dpoi_next_node;
 672           next1 = dpo1->dpoi_next_node;
 673
 674           /* Only process the HBH Option Header if explicitly configured to do so */
 675           if (PREDICT_FALSE
 676               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 677             {
 678               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 679                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 680             }
 681           /* Only process the HBH Option Header if explicitly configured to do so */
 682           if (PREDICT_FALSE
 683               (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 684             {
 685               next1 = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 686                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 687             }
 688
 689           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 690           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 691
 692           vlib_increment_combined_counter
 693             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 694           vlib_increment_combined_counter
 695             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 696
 697           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 698                                            to_next, n_left_to_next,
 699                                            pi0, pi1, next0, next1);
 700         }
 701
 702       while (n_left_from > 0 && n_left_to_next > 0)
 703         {
 704           ip_lookup_next_t next0;
 705           const load_balance_t *lb0;
 706           vlib_buffer_t *p0;
 707           u32 pi0, lbi0, hc0;
 708           const ip6_header_t *ip0;
 709           const dpo_id_t *dpo0;
 710
 711           pi0 = from[0];
 712           to_next[0] = pi0;
 713           from += 1;
 714           to_next += 1;
 715           n_left_to_next -= 1;
 716           n_left_from -= 1;
 717
 718           p0 = vlib_get_buffer (vm, pi0);
 719
 720           ip0 = vlib_buffer_get_current (p0);
 721           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 722
 723           lb0 = load_balance_get (lbi0);
 724
 725           hc0 = 0;
 726           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 727             {
 728               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 729                 {
 730                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 731                     vnet_buffer (p0)->ip.flow_hash >> 1;
 732                 }
 733               else
 734                 {
 735                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 736                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 737                 }
 738               dpo0 =
 739                 load_balance_get_fwd_bucket (lb0,
 740                                              (hc0 &
 741                                               lb0->lb_n_buckets_minus_1));
 742             }
 743           else
 744             {
 745               dpo0 = load_balance_get_bucket_i (lb0, 0);
 746             }
 747
 748           next0 = dpo0->dpoi_next_node;
 749           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 750
 751           /* Only process the HBH Option Header if explicitly configured to do so */
 752           if (PREDICT_FALSE
 753               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 754             {
 755               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 756                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 757             }
 758
 759           vlib_increment_combined_counter
 760             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 761
 762           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 763                                            to_next, n_left_to_next,
 764                                            pi0, next0);
 765         }
 766
 767       vlib_put_next_frame (vm, node, next, n_left_to_next);
 768     }
 769
 770   if (node->flags & VLIB_NODE_FLAG_TRACE)
 771     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 772
 773   return frame->n_vectors;
 774 }
 775
 776 /* *INDENT-OFF* */
 777 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 778 {
 779   .function = ip6_load_balance,
 780   .name = "ip6-load-balance",
 781   .vector_size = sizeof (u32),
 782   .sibling_of = "ip6-lookup",
 783   .format_trace = format_ip6_lookup_trace,
 784 };
 785 /* *INDENT-ON* */
 786
 787 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance);
 788
 789 typedef struct
 790 {
 791   /* Adjacency taken. */
 792   u32 adj_index;
 793   u32 flow_hash;
 794   u32 fib_index;
 795
 796   /* Packet data, possibly *after* rewrite. */
 797   u8 packet_data[128 - 1 * sizeof (u32)];
 798 }
 799 ip6_forward_next_trace_t;
 800
 801 u8 *
 802 format_ip6_forward_next_trace (u8 * s, va_list * args)
 803 {
 804   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 805   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 806   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 807   u32 indent = format_get_indent (s);
 808
 809   s = format (s, "%U%U",
 810               format_white_space, indent,
 811               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 812   return s;
 813 }
 814
 815 static u8 *
 816 format_ip6_lookup_trace (u8 * s, va_list * args)
 817 {
 818   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 819   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 820   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 821   u32 indent = format_get_indent (s);
 822
 823   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 824               t->fib_index, t->adj_index, t->flow_hash);
 825   s = format (s, "\n%U%U",
 826               format_white_space, indent,
 827               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 828   return s;
 829 }
 830
 831
 832 static u8 *
 833 format_ip6_rewrite_trace (u8 * s, va_list * args)
 834 {
 835   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 836   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 837   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 838   u32 indent = format_get_indent (s);
 839
 840   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 841               t->fib_index, t->adj_index, format_ip_adjacency,
 842               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 843   s = format (s, "\n%U%U",
 844               format_white_space, indent,
 845               format_ip_adjacency_packet_data,
 846               t->adj_index, t->packet_data, sizeof (t->packet_data));
 847   return s;
 848 }
 849
 850 /* Common trace function for all ip6-forward next nodes. */
 851 void
 852 ip6_forward_next_trace (vlib_main_t * vm,
 853                         vlib_node_runtime_t * node,
 854                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 855 {
 856   u32 *from, n_left;
 857   ip6_main_t *im = &ip6_main;
 858
 859   n_left = frame->n_vectors;
 860   from = vlib_frame_vector_args (frame);
 861
 862   while (n_left >= 4)
 863     {
 864       u32 bi0, bi1;
 865       vlib_buffer_t *b0, *b1;
 866       ip6_forward_next_trace_t *t0, *t1;
 867
 868       /* Prefetch next iteration. */
 869       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 870       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 871
 872       bi0 = from[0];
 873       bi1 = from[1];
 874
 875       b0 = vlib_get_buffer (vm, bi0);
 876       b1 = vlib_get_buffer (vm, bi1);
 877
 878       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 879         {
 880           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 881           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 882           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 883           t0->fib_index =
 884             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 885              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 886             vec_elt (im->fib_index_by_sw_if_index,
 887                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 888
 889           clib_memcpy_fast (t0->packet_data,
 890                             vlib_buffer_get_current (b0),
 891                             sizeof (t0->packet_data));
 892         }
 893       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 894         {
 895           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 896           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 897           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 898           t1->fib_index =
 899             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
 900              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
 901             vec_elt (im->fib_index_by_sw_if_index,
 902                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
 903
 904           clib_memcpy_fast (t1->packet_data,
 905                             vlib_buffer_get_current (b1),
 906                             sizeof (t1->packet_data));
 907         }
 908       from += 2;
 909       n_left -= 2;
 910     }
 911
 912   while (n_left >= 1)
 913     {
 914       u32 bi0;
 915       vlib_buffer_t *b0;
 916       ip6_forward_next_trace_t *t0;
 917
 918       bi0 = from[0];
 919
 920       b0 = vlib_get_buffer (vm, bi0);
 921
 922       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 923         {
 924           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 925           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 926           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 927           t0->fib_index =
 928             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 929              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 930             vec_elt (im->fib_index_by_sw_if_index,
 931                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 932
 933           clib_memcpy_fast (t0->packet_data,
 934                             vlib_buffer_get_current (b0),
 935                             sizeof (t0->packet_data));
 936         }
 937       from += 1;
 938       n_left -= 1;
 939     }
 940 }
 941
 942 /* Compute TCP/UDP/ICMP6 checksum in software. */
 943 u16
 944 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
 945                                    ip6_header_t * ip0, int *bogus_lengthp)
 946 {
 947   ip_csum_t sum0;
 948   u16 sum16, payload_length_host_byte_order;
 949   u32 i, n_this_buffer, n_bytes_left;
 950   u32 headers_size = sizeof (ip0[0]);
 951   void *data_this_buffer;
 952
 953   ASSERT (bogus_lengthp);
 954   *bogus_lengthp = 0;
 955
 956   /* Initialize checksum with ip header. */
 957   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
 958   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
 959   data_this_buffer = (void *) (ip0 + 1);
 960
 961   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
 962     {
 963       sum0 = ip_csum_with_carry (sum0,
 964                                  clib_mem_unaligned (&ip0->
 965                                                      src_address.as_uword[i],
 966                                                      uword));
 967       sum0 =
 968         ip_csum_with_carry (sum0,
 969                             clib_mem_unaligned (&ip0->dst_address.as_uword[i],
 970                                                 uword));
 971     }
 972
 973   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
 974    * or UDP-Ping packets */
 975   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 976     {
 977       u32 skip_bytes;
 978       ip6_hop_by_hop_ext_t *ext_hdr =
 979         (ip6_hop_by_hop_ext_t *) data_this_buffer;
 980
 981       /* validate really icmp6 next */
 982       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
 983               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
 984
 985       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
 986       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
 987
 988       payload_length_host_byte_order -= skip_bytes;
 989       headers_size += skip_bytes;
 990     }
 991
 992   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
 993   if (p0 && n_this_buffer + headers_size > p0->current_length)
 994     n_this_buffer =
 995       p0->current_length >
 996       headers_size ? p0->current_length - headers_size : 0;
 997   while (1)
 998     {
 999       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1000       n_bytes_left -= n_this_buffer;
1001       if (n_bytes_left == 0)
1002         break;
1003
1004       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1005         {
1006           *bogus_lengthp = 1;
1007           return 0xfefe;
1008         }
1009       p0 = vlib_get_buffer (vm, p0->next_buffer);
1010       data_this_buffer = vlib_buffer_get_current (p0);
1011       n_this_buffer = p0->current_length;
1012     }
1013
1014   sum16 = ~ip_csum_fold (sum0);
1015
1016   return sum16;
1017 }
1018
1019 u32
1020 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1021 {
1022   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1023   udp_header_t *udp0;
1024   u16 sum16;
1025   int bogus_length;
1026
1027   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1028   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1029           || ip0->protocol == IP_PROTOCOL_ICMP6
1030           || ip0->protocol == IP_PROTOCOL_UDP
1031           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1032
1033   udp0 = (void *) (ip0 + 1);
1034   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1035     {
1036       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1037                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1038       return p0->flags;
1039     }
1040
1041   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1042
1043   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1044                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1045
1046   return p0->flags;
1047 }
1048
1049 /**
1050  * @brief returns number of links on which src is reachable.
1051  */
1052 always_inline int
1053 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1054 {
1055   const load_balance_t *lb0;
1056   index_t lbi;
1057   u32 fib_index;
1058
1059   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1060                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1061   fib_index =
1062     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1063     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1064
1065   lbi = ip6_fib_table_fwding_lookup (im, fib_index, &i->src_address);
1066   lb0 = load_balance_get (lbi);
1067
1068   return (fib_urpf_check_size (lb0->lb_urpf));
1069 }
1070
1071 always_inline u8
1072 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1073                            u32 * udp_offset0)
1074 {
1075   u32 proto0;
1076   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1077   if (proto0 != IP_PROTOCOL_UDP)
1078     {
1079       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1080       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1081     }
1082   return proto0;
1083 }
1084
1085 /* *INDENT-OFF* */
1086 VNET_FEATURE_ARC_INIT (ip6_local) =
1087 {
1088   .arc_name  = "ip6-local",
1089   .start_nodes = VNET_FEATURES ("ip6-local"),
1090 };
1091 /* *INDENT-ON* */
1092
1093 static uword
1094 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1095                   vlib_frame_t * frame, int head_of_feature_arc)
1096 {
1097   ip6_main_t *im = &ip6_main;
1098   ip_lookup_main_t *lm = &im->lookup_main;
1099   ip_local_next_t next_index;
1100   u32 *from, *to_next, n_left_from, n_left_to_next;
1101   vlib_node_runtime_t *error_node =
1102     vlib_node_get_runtime (vm, ip6_input_node.index);
1103   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1104
1105   from = vlib_frame_vector_args (frame);
1106   n_left_from = frame->n_vectors;
1107   next_index = node->cached_next_index;
1108
1109   if (node->flags & VLIB_NODE_FLAG_TRACE)
1110     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1111
1112   while (n_left_from > 0)
1113     {
1114       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1115
1116       while (n_left_from >= 4 && n_left_to_next >= 2)
1117         {
1118           vlib_buffer_t *p0, *p1;
1119           ip6_header_t *ip0, *ip1;
1120           udp_header_t *udp0, *udp1;
1121           u32 pi0, ip_len0, udp_len0, flags0, next0;
1122           u32 pi1, ip_len1, udp_len1, flags1, next1;
1123           i32 len_diff0, len_diff1;
1124           u8 error0, type0, good_l4_csum0, is_tcp_udp0;
1125           u8 error1, type1, good_l4_csum1, is_tcp_udp1;
1126           u32 udp_offset0, udp_offset1;
1127
1128           pi0 = to_next[0] = from[0];
1129           pi1 = to_next[1] = from[1];
1130           from += 2;
1131           n_left_from -= 2;
1132           to_next += 2;
1133           n_left_to_next -= 2;
1134
1135           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1136
1137           p0 = vlib_get_buffer (vm, pi0);
1138           p1 = vlib_get_buffer (vm, pi1);
1139
1140           ip0 = vlib_buffer_get_current (p0);
1141           ip1 = vlib_buffer_get_current (p1);
1142
1143           if (head_of_feature_arc == 0)
1144             goto skip_checks;
1145
1146           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1147           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1148
1149           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1150           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1151
1152           flags0 = p0->flags;
1153           flags1 = p1->flags;
1154
1155           is_tcp_udp0 = ip6_next_proto_is_tcp_udp (p0, ip0, &udp_offset0);
1156           is_tcp_udp1 = ip6_next_proto_is_tcp_udp (p1, ip1, &udp_offset1);
1157
1158           good_l4_csum0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1159                            || (flags0 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1160                                || flags0 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1161             != 0;
1162           good_l4_csum1 = (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1163                            || (flags1 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1164                                || flags1 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1165             != 0;
1166           len_diff0 = 0;
1167           len_diff1 = 0;
1168
1169           if (PREDICT_TRUE (is_tcp_udp0))
1170             {
1171               udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0);
1172               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1173               good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP
1174                 && udp0->checksum == 0;
1175               /* Verify UDP length. */
1176               if (is_tcp_udp0 == IP_PROTOCOL_UDP)
1177                 {
1178                   ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1179                   udp_len0 = clib_net_to_host_u16 (udp0->length);
1180                   len_diff0 = ip_len0 - udp_len0;
1181                 }
1182             }
1183           if (PREDICT_TRUE (is_tcp_udp1))
1184             {
1185               udp1 = (udp_header_t *) ((u8 *) ip1 + udp_offset1);
1186               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1187               good_l4_csum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP
1188                 && udp1->checksum == 0;
1189               /* Verify UDP length. */
1190               if (is_tcp_udp1 == IP_PROTOCOL_UDP)
1191                 {
1192                   ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1193                   udp_len1 = clib_net_to_host_u16 (udp1->length);
1194                   len_diff1 = ip_len1 - udp_len1;
1195                 }
1196             }
1197
1198           good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1199           good_l4_csum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1200
1201           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1202           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1203
1204           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1205                              && !good_l4_csum0
1206                              && !(flags0 &
1207                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1208             {
1209               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1210               good_l4_csum0 =
1211                 (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1212             }
1213           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1214                              && !good_l4_csum1
1215                              && !(flags1 &
1216                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1217             {
1218               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1219               good_l4_csum1 =
1220                 (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1221             }
1222
1223           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1224           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1225           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1226
1227           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1228                   IP6_ERROR_UDP_CHECKSUM);
1229           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1230                   IP6_ERROR_ICMP_CHECKSUM);
1231           error0 = (!good_l4_csum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0);
1232           error1 = (!good_l4_csum1 ? IP6_ERROR_UDP_CHECKSUM + type1 : error1);
1233
1234           /* Drop packets from unroutable hosts. */
1235           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1236           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1237               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1238               !ip6_address_is_link_local_unicast (&ip0->src_address))
1239             {
1240               error0 = (!ip6_urpf_loose_check (im, p0, ip0)
1241                         ? IP6_ERROR_SRC_LOOKUP_MISS : error0);
1242             }
1243           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1244               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1245               !ip6_address_is_link_local_unicast (&ip1->src_address))
1246             {
1247               error1 = (!ip6_urpf_loose_check (im, p1, ip1)
1248                         ? IP6_ERROR_SRC_LOOKUP_MISS : error1);
1249             }
1250
1251           vnet_buffer (p0)->ip.fib_index =
1252             vnet_buffer (p0)->sw_if_index[VLIB_TX] != ~0 ?
1253             vnet_buffer (p0)->sw_if_index[VLIB_TX] :
1254             vnet_buffer (p0)->ip.fib_index;
1255
1256           vnet_buffer (p1)->ip.fib_index =
1257             vnet_buffer (p1)->sw_if_index[VLIB_TX] != ~0 ?
1258             vnet_buffer (p1)->sw_if_index[VLIB_TX] :
1259             vnet_buffer (p1)->ip.fib_index;
1260
1261         skip_checks:
1262
1263           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1264           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1265
1266           next0 =
1267             error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1268           next1 =
1269             error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1270
1271           p0->error = error_node->errors[error0];
1272           p1->error = error_node->errors[error1];
1273
1274           if (head_of_feature_arc)
1275             {
1276               if (PREDICT_TRUE (error0 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1277                 vnet_feature_arc_start (arc_index,
1278                                         vnet_buffer (p0)->sw_if_index
1279                                         [VLIB_RX], &next0, p0);
1280               if (PREDICT_TRUE (error1 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1281                 vnet_feature_arc_start (arc_index,
1282                                         vnet_buffer (p1)->sw_if_index
1283                                         [VLIB_RX], &next1, p1);
1284             }
1285
1286           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1287                                            to_next, n_left_to_next,
1288                                            pi0, pi1, next0, next1);
1289         }
1290
1291       while (n_left_from > 0 && n_left_to_next > 0)
1292         {
1293           vlib_buffer_t *p0;
1294           ip6_header_t *ip0;
1295           udp_header_t *udp0;
1296           u32 pi0, ip_len0, udp_len0, flags0, next0;
1297           i32 len_diff0;
1298           u8 error0, type0, good_l4_csum0;
1299           u32 udp_offset0;
1300           u8 is_tcp_udp0;
1301
1302           pi0 = to_next[0] = from[0];
1303           from += 1;
1304           n_left_from -= 1;
1305           to_next += 1;
1306           n_left_to_next -= 1;
1307
1308           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1309
1310           p0 = vlib_get_buffer (vm, pi0);
1311           ip0 = vlib_buffer_get_current (p0);
1312
1313           if (head_of_feature_arc == 0)
1314             goto skip_check;
1315
1316           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1317
1318           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1319           flags0 = p0->flags;
1320           is_tcp_udp0 = ip6_next_proto_is_tcp_udp (p0, ip0, &udp_offset0);
1321           good_l4_csum0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1322                            || (flags0 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1323                                || flags0 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1324             != 0;
1325
1326           len_diff0 = 0;
1327           if (PREDICT_TRUE (is_tcp_udp0))
1328             {
1329               udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0);
1330               /* Don't verify UDP checksum for packets with explicit zero
1331                * checksum. */
1332               good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP
1333                 && udp0->checksum == 0;
1334               /* Verify UDP length. */
1335               if (is_tcp_udp0 == IP_PROTOCOL_UDP)
1336                 {
1337                   ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1338                   udp_len0 = clib_net_to_host_u16 (udp0->length);
1339                   len_diff0 = ip_len0 - udp_len0;
1340                 }
1341             }
1342
1343           good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1344           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1345
1346           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1347                              && !good_l4_csum0
1348                              && !(flags0 &
1349                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1350             {
1351               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1352               good_l4_csum0 =
1353                 (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1354             }
1355
1356           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1357           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1358
1359           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1360                   IP6_ERROR_UDP_CHECKSUM);
1361           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1362                   IP6_ERROR_ICMP_CHECKSUM);
1363           error0 = (!good_l4_csum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0);
1364
1365           /* If this is a neighbor solicitation (ICMP), skip src RPF check */
1366           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1367               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1368               !ip6_address_is_link_local_unicast (&ip0->src_address))
1369             {
1370               error0 = (!ip6_urpf_loose_check (im, p0, ip0)
1371                         ? IP6_ERROR_SRC_LOOKUP_MISS : error0);
1372             }
1373
1374           vnet_buffer (p0)->ip.fib_index =
1375             vnet_buffer (p0)->sw_if_index[VLIB_TX] != ~0 ?
1376             vnet_buffer (p0)->sw_if_index[VLIB_TX] :
1377             vnet_buffer (p0)->ip.fib_index;
1378
1379         skip_check:
1380
1381           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1382           next0 =
1383             error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1384
1385           p0->error = error_node->errors[error0];
1386
1387           if (head_of_feature_arc)
1388             {
1389               if (PREDICT_TRUE (error0 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1390                 vnet_feature_arc_start (arc_index,
1391                                         vnet_buffer (p0)->sw_if_index
1392                                         [VLIB_RX], &next0, p0);
1393             }
1394
1395           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1396                                            to_next, n_left_to_next,
1397                                            pi0, next0);
1398         }
1399
1400       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1401     }
1402
1403   return frame->n_vectors;
1404 }
1405
1406 static uword
1407 ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1408 {
1409   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1410 }
1411
1412 /* *INDENT-OFF* */
1413 VLIB_REGISTER_NODE (ip6_local_node, static) =
1414 {
1415   .function = ip6_local,
1416   .name = "ip6-local",
1417   .vector_size = sizeof (u32),
1418   .format_trace = format_ip6_forward_next_trace,
1419   .n_next_nodes = IP_LOCAL_N_NEXT,
1420   .next_nodes =
1421   {
1422     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1423     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1424     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1425     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1426     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-reassembly",
1427   },
1428 };
1429 /* *INDENT-ON* */
1430
1431 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local);
1432
1433
1434 static uword
1435 ip6_local_end_of_arc (vlib_main_t * vm,
1436                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1437 {
1438   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1439 }
1440
1441 /* *INDENT-OFF* */
1442 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node,static) = {
1443   .function = ip6_local_end_of_arc,
1444   .name = "ip6-local-end-of-arc",
1445   .vector_size = sizeof (u32),
1446
1447   .format_trace = format_ip6_forward_next_trace,
1448   .sibling_of = "ip6-local",
1449 };
1450
1451 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_end_of_arc_node, ip6_local_end_of_arc)
1452
1453 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1454   .arc_name = "ip6-local",
1455   .node_name = "ip6-local-end-of-arc",
1456   .runs_before = 0, /* not before any other features */
1457 };
1458 /* *INDENT-ON* */
1459
1460 void
1461 ip6_register_protocol (u32 protocol, u32 node_index)
1462 {
1463   vlib_main_t *vm = vlib_get_main ();
1464   ip6_main_t *im = &ip6_main;
1465   ip_lookup_main_t *lm = &im->lookup_main;
1466
1467   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1468   lm->local_next_by_ip_protocol[protocol] =
1469     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1470 }
1471
1472 clib_error_t *
1473 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index,
1474                     u8 refresh)
1475 {
1476   vnet_main_t *vnm = vnet_get_main ();
1477   ip6_main_t *im = &ip6_main;
1478   icmp6_neighbor_solicitation_header_t *h;
1479   ip6_address_t *src;
1480   ip_interface_address_t *ia;
1481   ip_adjacency_t *adj;
1482   vnet_hw_interface_t *hi;
1483   vnet_sw_interface_t *si;
1484   vlib_buffer_t *b;
1485   adj_index_t ai;
1486   u32 bi = 0;
1487   int bogus_length;
1488
1489   si = vnet_get_sw_interface (vnm, sw_if_index);
1490
1491   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1492     {
1493       return clib_error_return (0, "%U: interface %U down",
1494                                 format_ip6_address, dst,
1495                                 format_vnet_sw_if_index_name, vnm,
1496                                 sw_if_index);
1497     }
1498
1499   src =
1500     ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1501   if (!src)
1502     {
1503       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1504       return clib_error_return
1505         (0, "no matching interface address for destination %U (interface %U)",
1506          format_ip6_address, dst,
1507          format_vnet_sw_if_index_name, vnm, sw_if_index);
1508     }
1509
1510   h =
1511     vlib_packet_template_get_packet (vm,
1512                                      &im->discover_neighbor_packet_template,
1513                                      &bi);
1514   if (!h)
1515     return clib_error_return (0, "ICMP6 NS packet allocation failed");
1516
1517   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1518
1519   /* Destination address is a solicited node multicast address.  We need to fill in
1520      the low 24 bits with low 24 bits of target's address. */
1521   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1522   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1523   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1524
1525   h->ip.src_address = src[0];
1526   h->neighbor.target_address = dst[0];
1527
1528   if (PREDICT_FALSE (!hi->hw_address))
1529     {
1530       return clib_error_return (0, "%U: interface %U do not support ip probe",
1531                                 format_ip6_address, dst,
1532                                 format_vnet_sw_if_index_name, vnm,
1533                                 sw_if_index);
1534     }
1535
1536   clib_memcpy_fast (h->link_layer_option.ethernet_address, hi->hw_address,
1537                     vec_len (hi->hw_address));
1538
1539   h->neighbor.icmp.checksum =
1540     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1541   ASSERT (bogus_length == 0);
1542
1543   b = vlib_get_buffer (vm, bi);
1544   vnet_buffer (b)->sw_if_index[VLIB_RX] =
1545     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1546
1547   /* Add encapsulation string for software interface (e.g. ethernet header). */
1548   ip46_address_t nh = {
1549     .ip6 = *dst,
1550   };
1551
1552   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP6,
1553                             VNET_LINK_IP6, &nh, sw_if_index);
1554   adj = adj_get (ai);
1555
1556   /* Peer has been previously resolved, retrieve glean adj instead */
1557   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE && refresh == 0)
1558     {
1559       adj_unlock (ai);
1560       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP6,
1561                                   VNET_LINK_IP6, sw_if_index, &nh);
1562       adj = adj_get (ai);
1563     }
1564
1565   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1566   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1567
1568   {
1569     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
1570     u32 *to_next = vlib_frame_vector_args (f);
1571     to_next[0] = bi;
1572     f->n_vectors = 1;
1573     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1574   }
1575
1576   adj_unlock (ai);
1577   return /* no error */ 0;
1578 }
1579
1580 typedef enum
1581 {
1582   IP6_REWRITE_NEXT_DROP,
1583   IP6_REWRITE_NEXT_ICMP_ERROR,
1584   IP6_REWRITE_NEXT_FRAGMENT,
1585   IP6_REWRITE_N_NEXT            /* Last */
1586 } ip6_rewrite_next_t;
1587
1588 /**
1589  * This bits of an IPv6 address to mask to construct a multicast
1590  * MAC address
1591  */
1592 #define IP6_MCAST_ADDR_MASK 0xffffffff
1593
1594 always_inline void
1595 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1596                u16 adj_packet_bytes, bool is_locally_generated,
1597                u32 * next, u32 * error)
1598 {
1599   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1600     {
1601       if (is_locally_generated)
1602         {
1603           /* IP fragmentation */
1604           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1605                                    IP6_FRAG_NEXT_IP6_REWRITE, 0);
1606           *next = IP6_REWRITE_NEXT_FRAGMENT;
1607           *error = IP6_ERROR_MTU_EXCEEDED;
1608         }
1609       else
1610         {
1611           *error = IP6_ERROR_MTU_EXCEEDED;
1612           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1613                                        adj_packet_bytes);
1614           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1615         }
1616     }
1617 }
1618
1619 always_inline uword
1620 ip6_rewrite_inline (vlib_main_t * vm,
1621                     vlib_node_runtime_t * node,
1622                     vlib_frame_t * frame,
1623                     int do_counters, int is_midchain, int is_mcast)
1624 {
1625   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1626   u32 *from = vlib_frame_vector_args (frame);
1627   u32 n_left_from, n_left_to_next, *to_next, next_index;
1628   vlib_node_runtime_t *error_node =
1629     vlib_node_get_runtime (vm, ip6_input_node.index);
1630
1631   n_left_from = frame->n_vectors;
1632   next_index = node->cached_next_index;
1633   u32 thread_index = vm->thread_index;
1634
1635   while (n_left_from > 0)
1636     {
1637       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1638
1639       while (n_left_from >= 4 && n_left_to_next >= 2)
1640         {
1641           ip_adjacency_t *adj0, *adj1;
1642           vlib_buffer_t *p0, *p1;
1643           ip6_header_t *ip0, *ip1;
1644           u32 pi0, rw_len0, next0, error0, adj_index0;
1645           u32 pi1, rw_len1, next1, error1, adj_index1;
1646           u32 tx_sw_if_index0, tx_sw_if_index1;
1647           bool is_locally_originated0, is_locally_originated1;
1648
1649           /* Prefetch next iteration. */
1650           {
1651             vlib_buffer_t *p2, *p3;
1652
1653             p2 = vlib_get_buffer (vm, from[2]);
1654             p3 = vlib_get_buffer (vm, from[3]);
1655
1656             vlib_prefetch_buffer_header (p2, LOAD);
1657             vlib_prefetch_buffer_header (p3, LOAD);
1658
1659             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1660             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1661
1662             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1663             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1664           }
1665
1666           pi0 = to_next[0] = from[0];
1667           pi1 = to_next[1] = from[1];
1668
1669           from += 2;
1670           n_left_from -= 2;
1671           to_next += 2;
1672           n_left_to_next -= 2;
1673
1674           p0 = vlib_get_buffer (vm, pi0);
1675           p1 = vlib_get_buffer (vm, pi1);
1676
1677           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1678           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1679
1680           ip0 = vlib_buffer_get_current (p0);
1681           ip1 = vlib_buffer_get_current (p1);
1682
1683           error0 = error1 = IP6_ERROR_NONE;
1684           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1685
1686           is_locally_originated0 =
1687             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1688           if (PREDICT_TRUE (!is_locally_originated0))
1689             {
1690               i32 hop_limit0 = ip0->hop_limit;
1691
1692               /* Input node should have reject packets with hop limit 0. */
1693               ASSERT (ip0->hop_limit > 0);
1694
1695               hop_limit0 -= 1;
1696
1697               ip0->hop_limit = hop_limit0;
1698
1699               /*
1700                * If the hop count drops below 1 when forwarding, generate
1701                * an ICMP response.
1702                */
1703               if (PREDICT_FALSE (hop_limit0 <= 0))
1704                 {
1705                   error0 = IP6_ERROR_TIME_EXPIRED;
1706                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1707                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1708                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1709                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1710                                                0);
1711                 }
1712             }
1713           else
1714             {
1715               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1716             }
1717           is_locally_originated1 =
1718             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1719           if (PREDICT_TRUE (!is_locally_originated1))
1720             {
1721               i32 hop_limit1 = ip1->hop_limit;
1722
1723               /* Input node should have reject packets with hop limit 0. */
1724               ASSERT (ip1->hop_limit > 0);
1725
1726               hop_limit1 -= 1;
1727
1728               ip1->hop_limit = hop_limit1;
1729
1730               /*
1731                * If the hop count drops below 1 when forwarding, generate
1732                * an ICMP response.
1733                */
1734               if (PREDICT_FALSE (hop_limit1 <= 0))
1735                 {
1736                   error1 = IP6_ERROR_TIME_EXPIRED;
1737                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1738                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1739                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1740                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1741                                                0);
1742                 }
1743             }
1744           else
1745             {
1746               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1747             }
1748           adj0 = adj_get (adj_index0);
1749           adj1 = adj_get (adj_index1);
1750
1751           rw_len0 = adj0[0].rewrite_header.data_bytes;
1752           rw_len1 = adj1[0].rewrite_header.data_bytes;
1753           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1754           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1755
1756           if (do_counters)
1757             {
1758               vlib_increment_combined_counter
1759                 (&adjacency_counters,
1760                  thread_index, adj_index0, 1,
1761                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1762               vlib_increment_combined_counter
1763                 (&adjacency_counters,
1764                  thread_index, adj_index1, 1,
1765                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1766             }
1767
1768           /* Check MTU of outgoing interface. */
1769           ip6_mtu_check (p0, clib_net_to_host_u16 (ip0->payload_length) +
1770                          sizeof (ip6_header_t),
1771                          adj0[0].rewrite_header.max_l3_packet_bytes,
1772                          is_locally_originated0, &next0, &error0);
1773           ip6_mtu_check (p1, clib_net_to_host_u16 (ip1->payload_length) +
1774                          sizeof (ip6_header_t),
1775                          adj1[0].rewrite_header.max_l3_packet_bytes,
1776                          is_locally_originated1, &next1, &error1);
1777
1778           /* Don't adjust the buffer for hop count issue; icmp-error node
1779            * wants to see the IP headerr */
1780           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1781             {
1782               p0->current_data -= rw_len0;
1783               p0->current_length += rw_len0;
1784
1785               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1786               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1787               next0 = adj0[0].rewrite_header.next_index;
1788
1789               if (PREDICT_FALSE
1790                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1791                 vnet_feature_arc_start (lm->output_feature_arc_index,
1792                                         tx_sw_if_index0, &next0, p0);
1793             }
1794           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1795             {
1796               p1->current_data -= rw_len1;
1797               p1->current_length += rw_len1;
1798
1799               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1800               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1801               next1 = adj1[0].rewrite_header.next_index;
1802
1803               if (PREDICT_FALSE
1804                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1805                 vnet_feature_arc_start (lm->output_feature_arc_index,
1806                                         tx_sw_if_index1, &next1, p1);
1807             }
1808
1809           /* Guess we are only writing on simple Ethernet header. */
1810           vnet_rewrite_two_headers (adj0[0], adj1[0],
1811                                     ip0, ip1, sizeof (ethernet_header_t));
1812
1813           if (is_midchain)
1814             {
1815               adj0->sub_type.midchain.fixup_func
1816                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1817               adj1->sub_type.midchain.fixup_func
1818                 (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1819             }
1820           if (is_mcast)
1821             {
1822               /*
1823                * copy bytes from the IP address into the MAC rewrite
1824                */
1825               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1826                                           adj0->
1827                                           rewrite_header.dst_mcast_offset,
1828                                           &ip0->dst_address.as_u32[3],
1829                                           (u8 *) ip0);
1830               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1831                                           adj1->
1832                                           rewrite_header.dst_mcast_offset,
1833                                           &ip1->dst_address.as_u32[3],
1834                                           (u8 *) ip1);
1835             }
1836
1837           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1838                                            to_next, n_left_to_next,
1839                                            pi0, pi1, next0, next1);
1840         }
1841
1842       while (n_left_from > 0 && n_left_to_next > 0)
1843         {
1844           ip_adjacency_t *adj0;
1845           vlib_buffer_t *p0;
1846           ip6_header_t *ip0;
1847           u32 pi0, rw_len0;
1848           u32 adj_index0, next0, error0;
1849           u32 tx_sw_if_index0;
1850           bool is_locally_originated0;
1851
1852           pi0 = to_next[0] = from[0];
1853
1854           p0 = vlib_get_buffer (vm, pi0);
1855
1856           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1857
1858           adj0 = adj_get (adj_index0);
1859
1860           ip0 = vlib_buffer_get_current (p0);
1861
1862           error0 = IP6_ERROR_NONE;
1863           next0 = IP6_REWRITE_NEXT_DROP;
1864
1865           /* Check hop limit */
1866           is_locally_originated0 =
1867             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1868           if (PREDICT_TRUE (!is_locally_originated0))
1869             {
1870               i32 hop_limit0 = ip0->hop_limit;
1871
1872               ASSERT (ip0->hop_limit > 0);
1873
1874               hop_limit0 -= 1;
1875
1876               ip0->hop_limit = hop_limit0;
1877
1878               if (PREDICT_FALSE (hop_limit0 <= 0))
1879                 {
1880                   /*
1881                    * If the hop count drops below 1 when forwarding, generate
1882                    * an ICMP response.
1883                    */
1884                   error0 = IP6_ERROR_TIME_EXPIRED;
1885                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1886                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1887                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1888                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1889                                                0);
1890                 }
1891             }
1892           else
1893             {
1894               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1895             }
1896
1897           /* Guess we are only writing on simple Ethernet header. */
1898           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
1899
1900           /* Update packet buffer attributes/set output interface. */
1901           rw_len0 = adj0[0].rewrite_header.data_bytes;
1902           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1903
1904           if (do_counters)
1905             {
1906               vlib_increment_combined_counter
1907                 (&adjacency_counters,
1908                  thread_index, adj_index0, 1,
1909                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1910             }
1911
1912           /* Check MTU of outgoing interface. */
1913           ip6_mtu_check (p0, clib_net_to_host_u16 (ip0->payload_length) +
1914                          sizeof (ip6_header_t),
1915                          adj0[0].rewrite_header.max_l3_packet_bytes,
1916                          is_locally_originated0, &next0, &error0);
1917
1918           /* Don't adjust the buffer for hop count issue; icmp-error node
1919            * wants to see the IP header */
1920           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1921             {
1922               p0->current_data -= rw_len0;
1923               p0->current_length += rw_len0;
1924
1925               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1926
1927               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1928               next0 = adj0[0].rewrite_header.next_index;
1929
1930               if (PREDICT_FALSE
1931                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1932                 vnet_feature_arc_start (lm->output_feature_arc_index,
1933                                         tx_sw_if_index0, &next0, p0);
1934             }
1935
1936           if (is_midchain)
1937             {
1938               adj0->sub_type.midchain.fixup_func
1939                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1940             }
1941           if (is_mcast)
1942             {
1943               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1944                                           adj0->
1945                                           rewrite_header.dst_mcast_offset,
1946                                           &ip0->dst_address.as_u32[3],
1947                                           (u8 *) ip0);
1948             }
1949
1950           p0->error = error_node->errors[error0];
1951
1952           from += 1;
1953           n_left_from -= 1;
1954           to_next += 1;
1955           n_left_to_next -= 1;
1956
1957           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1958                                            to_next, n_left_to_next,
1959                                            pi0, next0);
1960         }
1961
1962       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1963     }
1964
1965   /* Need to do trace after rewrites to pick up new packet data. */
1966   if (node->flags & VLIB_NODE_FLAG_TRACE)
1967     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1968
1969   return frame->n_vectors;
1970 }
1971
1972 static uword
1973 ip6_rewrite (vlib_main_t * vm,
1974              vlib_node_runtime_t * node, vlib_frame_t * frame)
1975 {
1976   if (adj_are_counters_enabled ())
1977     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
1978   else
1979     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
1980 }
1981
1982 static uword
1983 ip6_rewrite_bcast (vlib_main_t * vm,
1984                    vlib_node_runtime_t * node, vlib_frame_t * frame)
1985 {
1986   if (adj_are_counters_enabled ())
1987     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
1988   else
1989     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
1990 }
1991
1992 static uword
1993 ip6_rewrite_mcast (vlib_main_t * vm,
1994                    vlib_node_runtime_t * node, vlib_frame_t * frame)
1995 {
1996   if (adj_are_counters_enabled ())
1997     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
1998   else
1999     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2000 }
2001
2002 static uword
2003 ip6_midchain (vlib_main_t * vm,
2004               vlib_node_runtime_t * node, vlib_frame_t * frame)
2005 {
2006   if (adj_are_counters_enabled ())
2007     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2008   else
2009     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2010 }
2011
2012 static uword
2013 ip6_mcast_midchain (vlib_main_t * vm,
2014                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2015 {
2016   if (adj_are_counters_enabled ())
2017     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2018   else
2019     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2020 }
2021
2022 /* *INDENT-OFF* */
2023 VLIB_REGISTER_NODE (ip6_midchain_node) =
2024 {
2025   .function = ip6_midchain,
2026   .name = "ip6-midchain",
2027   .vector_size = sizeof (u32),
2028   .format_trace = format_ip6_forward_next_trace,
2029   .sibling_of = "ip6-rewrite",
2030   };
2031
2032 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain);
2033
2034 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2035 {
2036   .function = ip6_rewrite,
2037   .name = "ip6-rewrite",
2038   .vector_size = sizeof (u32),
2039   .format_trace = format_ip6_rewrite_trace,
2040   .n_next_nodes = IP6_REWRITE_N_NEXT,
2041   .next_nodes =
2042   {
2043     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2044     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2045     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2046   },
2047 };
2048
2049 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite);
2050
2051 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2052   .function = ip6_rewrite_bcast,
2053   .name = "ip6-rewrite-bcast",
2054   .vector_size = sizeof (u32),
2055
2056   .format_trace = format_ip6_rewrite_trace,
2057   .sibling_of = "ip6-rewrite",
2058 };
2059 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_bcast_node, ip6_rewrite_bcast)
2060
2061 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2062 {
2063   .function = ip6_rewrite_mcast,
2064   .name = "ip6-rewrite-mcast",
2065   .vector_size = sizeof (u32),
2066   .format_trace = format_ip6_rewrite_trace,
2067   .sibling_of = "ip6-rewrite",
2068 };
2069
2070 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast);
2071
2072 VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) =
2073 {
2074   .function = ip6_mcast_midchain,
2075   .name = "ip6-mcast-midchain",
2076   .vector_size = sizeof (u32),
2077   .format_trace = format_ip6_rewrite_trace,
2078   .sibling_of = "ip6-rewrite",
2079 };
2080
2081 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain);
2082 /* *INDENT-ON* */
2083
2084 /*
2085  * Hop-by-Hop handling
2086  */
2087 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2088
2089 #define foreach_ip6_hop_by_hop_error \
2090 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2091 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2092 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2093
2094 /* *INDENT-OFF* */
2095 typedef enum
2096 {
2097 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2098   foreach_ip6_hop_by_hop_error
2099 #undef _
2100   IP6_HOP_BY_HOP_N_ERROR,
2101 } ip6_hop_by_hop_error_t;
2102 /* *INDENT-ON* */
2103
2104 /*
2105  * Primary h-b-h handler trace support
2106  * We work pretty hard on the problem for obvious reasons
2107  */
2108 typedef struct
2109 {
2110   u32 next_index;
2111   u32 trace_len;
2112   u8 option_data[256];
2113 } ip6_hop_by_hop_trace_t;
2114
2115 vlib_node_registration_t ip6_hop_by_hop_node;
2116
2117 static char *ip6_hop_by_hop_error_strings[] = {
2118 #define _(sym,string) string,
2119   foreach_ip6_hop_by_hop_error
2120 #undef _
2121 };
2122
2123 u8 *
2124 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2125 {
2126   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2127   int total_len = va_arg (*args, int);
2128   ip6_hop_by_hop_option_t *opt0, *limit0;
2129   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2130   u8 type0;
2131
2132   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2133               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2134
2135   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2136   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2137
2138   while (opt0 < limit0)
2139     {
2140       type0 = opt0->type;
2141       switch (type0)
2142         {
2143         case 0:         /* Pad, just stop */
2144           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2145           break;
2146
2147         default:
2148           if (hm->trace[type0])
2149             {
2150               s = (*hm->trace[type0]) (s, opt0);
2151             }
2152           else
2153             {
2154               s =
2155                 format (s, "\n    unrecognized option %d length %d", type0,
2156                         opt0->length);
2157             }
2158           opt0 =
2159             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2160                                          sizeof (ip6_hop_by_hop_option_t));
2161           break;
2162         }
2163     }
2164   return s;
2165 }
2166
2167 static u8 *
2168 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2169 {
2170   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2171   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2172   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2173   ip6_hop_by_hop_header_t *hbh0;
2174   ip6_hop_by_hop_option_t *opt0, *limit0;
2175   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2176
2177   u8 type0;
2178
2179   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2180
2181   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2182               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2183
2184   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2185   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2186
2187   while (opt0 < limit0)
2188     {
2189       type0 = opt0->type;
2190       switch (type0)
2191         {
2192         case 0:         /* Pad, just stop */
2193           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2194           break;
2195
2196         default:
2197           if (hm->trace[type0])
2198             {
2199               s = (*hm->trace[type0]) (s, opt0);
2200             }
2201           else
2202             {
2203               s =
2204                 format (s, "\n    unrecognized option %d length %d", type0,
2205                         opt0->length);
2206             }
2207           opt0 =
2208             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2209                                          sizeof (ip6_hop_by_hop_option_t));
2210           break;
2211         }
2212     }
2213   return s;
2214 }
2215
2216 always_inline u8
2217 ip6_scan_hbh_options (vlib_buffer_t * b0,
2218                       ip6_header_t * ip0,
2219                       ip6_hop_by_hop_header_t * hbh0,
2220                       ip6_hop_by_hop_option_t * opt0,
2221                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2222 {
2223   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2224   u8 type0;
2225   u8 error0 = 0;
2226
2227   while (opt0 < limit0)
2228     {
2229       type0 = opt0->type;
2230       switch (type0)
2231         {
2232         case 0:         /* Pad1 */
2233           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2234           continue;
2235         case 1:         /* PadN */
2236           break;
2237         default:
2238           if (hm->options[type0])
2239             {
2240               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2241                 {
2242                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2243                   return (error0);
2244                 }
2245             }
2246           else
2247             {
2248               /* Unrecognized mandatory option, check the two high order bits */
2249               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2250                 {
2251                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2252                   break;
2253                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2254                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2255                   *next0 = IP_LOOKUP_NEXT_DROP;
2256                   break;
2257                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2258                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2259                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2260                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2261                                                ICMP6_parameter_problem_unrecognized_option,
2262                                                (u8 *) opt0 - (u8 *) ip0);
2263                   break;
2264                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2265                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2266                   if (!ip6_address_is_multicast (&ip0->dst_address))
2267                     {
2268                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2269                       icmp6_error_set_vnet_buffer (b0,
2270                                                    ICMP6_parameter_problem,
2271                                                    ICMP6_parameter_problem_unrecognized_option,
2272                                                    (u8 *) opt0 - (u8 *) ip0);
2273                     }
2274                   else
2275                     {
2276                       *next0 = IP_LOOKUP_NEXT_DROP;
2277                     }
2278                   break;
2279                 }
2280               return (error0);
2281             }
2282         }
2283       opt0 =
2284         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2285                                      sizeof (ip6_hop_by_hop_option_t));
2286     }
2287   return (error0);
2288 }
2289
2290 /*
2291  * Process the Hop-by-Hop Options header
2292  */
2293 static uword
2294 ip6_hop_by_hop (vlib_main_t * vm,
2295                 vlib_node_runtime_t * node, vlib_frame_t * frame)
2296 {
2297   vlib_node_runtime_t *error_node =
2298     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2299   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2300   u32 n_left_from, *from, *to_next;
2301   ip_lookup_next_t next_index;
2302
2303   from = vlib_frame_vector_args (frame);
2304   n_left_from = frame->n_vectors;
2305   next_index = node->cached_next_index;
2306
2307   while (n_left_from > 0)
2308     {
2309       u32 n_left_to_next;
2310
2311       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2312
2313       while (n_left_from >= 4 && n_left_to_next >= 2)
2314         {
2315           u32 bi0, bi1;
2316           vlib_buffer_t *b0, *b1;
2317           u32 next0, next1;
2318           ip6_header_t *ip0, *ip1;
2319           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2320           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2321           u8 error0 = 0, error1 = 0;
2322
2323           /* Prefetch next iteration. */
2324           {
2325             vlib_buffer_t *p2, *p3;
2326
2327             p2 = vlib_get_buffer (vm, from[2]);
2328             p3 = vlib_get_buffer (vm, from[3]);
2329
2330             vlib_prefetch_buffer_header (p2, LOAD);
2331             vlib_prefetch_buffer_header (p3, LOAD);
2332
2333             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2334             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2335           }
2336
2337           /* Speculatively enqueue b0, b1 to the current next frame */
2338           to_next[0] = bi0 = from[0];
2339           to_next[1] = bi1 = from[1];
2340           from += 2;
2341           to_next += 2;
2342           n_left_from -= 2;
2343           n_left_to_next -= 2;
2344
2345           b0 = vlib_get_buffer (vm, bi0);
2346           b1 = vlib_get_buffer (vm, bi1);
2347
2348           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2349           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2350           ip_adjacency_t *adj0 = adj_get (adj_index0);
2351           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2352           ip_adjacency_t *adj1 = adj_get (adj_index1);
2353
2354           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2355           next0 = adj0->lookup_next_index;
2356           next1 = adj1->lookup_next_index;
2357
2358           ip0 = vlib_buffer_get_current (b0);
2359           ip1 = vlib_buffer_get_current (b1);
2360           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2361           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2362           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2363           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2364           limit0 =
2365             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2366                                          ((hbh0->length + 1) << 3));
2367           limit1 =
2368             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2369                                          ((hbh1->length + 1) << 3));
2370
2371           /*
2372            * Basic validity checks
2373            */
2374           if ((hbh0->length + 1) << 3 >
2375               clib_net_to_host_u16 (ip0->payload_length))
2376             {
2377               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2378               next0 = IP_LOOKUP_NEXT_DROP;
2379               goto outdual;
2380             }
2381           /* Scan the set of h-b-h options, process ones that we understand */
2382           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2383
2384           if ((hbh1->length + 1) << 3 >
2385               clib_net_to_host_u16 (ip1->payload_length))
2386             {
2387               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2388               next1 = IP_LOOKUP_NEXT_DROP;
2389               goto outdual;
2390             }
2391           /* Scan the set of h-b-h options, process ones that we understand */
2392           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2393
2394         outdual:
2395           /* Has the classifier flagged this buffer for special treatment? */
2396           if (PREDICT_FALSE
2397               ((error0 == 0)
2398                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2399             next0 = hm->next_override;
2400
2401           /* Has the classifier flagged this buffer for special treatment? */
2402           if (PREDICT_FALSE
2403               ((error1 == 0)
2404                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2405             next1 = hm->next_override;
2406
2407           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2408             {
2409               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2410                 {
2411                   ip6_hop_by_hop_trace_t *t =
2412                     vlib_add_trace (vm, node, b0, sizeof (*t));
2413                   u32 trace_len = (hbh0->length + 1) << 3;
2414                   t->next_index = next0;
2415                   /* Capture the h-b-h option verbatim */
2416                   trace_len =
2417                     trace_len <
2418                     ARRAY_LEN (t->option_data) ? trace_len :
2419                     ARRAY_LEN (t->option_data);
2420                   t->trace_len = trace_len;
2421                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2422                 }
2423               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2424                 {
2425                   ip6_hop_by_hop_trace_t *t =
2426                     vlib_add_trace (vm, node, b1, sizeof (*t));
2427                   u32 trace_len = (hbh1->length + 1) << 3;
2428                   t->next_index = next1;
2429                   /* Capture the h-b-h option verbatim */
2430                   trace_len =
2431                     trace_len <
2432                     ARRAY_LEN (t->option_data) ? trace_len :
2433                     ARRAY_LEN (t->option_data);
2434                   t->trace_len = trace_len;
2435                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2436                 }
2437
2438             }
2439
2440           b0->error = error_node->errors[error0];
2441           b1->error = error_node->errors[error1];
2442
2443           /* verify speculative enqueue, maybe switch current next frame */
2444           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2445                                            n_left_to_next, bi0, bi1, next0,
2446                                            next1);
2447         }
2448
2449       while (n_left_from > 0 && n_left_to_next > 0)
2450         {
2451           u32 bi0;
2452           vlib_buffer_t *b0;
2453           u32 next0;
2454           ip6_header_t *ip0;
2455           ip6_hop_by_hop_header_t *hbh0;
2456           ip6_hop_by_hop_option_t *opt0, *limit0;
2457           u8 error0 = 0;
2458
2459           /* Speculatively enqueue b0 to the current next frame */
2460           bi0 = from[0];
2461           to_next[0] = bi0;
2462           from += 1;
2463           to_next += 1;
2464           n_left_from -= 1;
2465           n_left_to_next -= 1;
2466
2467           b0 = vlib_get_buffer (vm, bi0);
2468           /*
2469            * Default use the next_index from the adjacency.
2470            * A HBH option rarely redirects to a different node
2471            */
2472           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2473           ip_adjacency_t *adj0 = adj_get (adj_index0);
2474           next0 = adj0->lookup_next_index;
2475
2476           ip0 = vlib_buffer_get_current (b0);
2477           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2478           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2479           limit0 =
2480             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2481                                          ((hbh0->length + 1) << 3));
2482
2483           /*
2484            * Basic validity checks
2485            */
2486           if ((hbh0->length + 1) << 3 >
2487               clib_net_to_host_u16 (ip0->payload_length))
2488             {
2489               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2490               next0 = IP_LOOKUP_NEXT_DROP;
2491               goto out0;
2492             }
2493
2494           /* Scan the set of h-b-h options, process ones that we understand */
2495           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2496
2497         out0:
2498           /* Has the classifier flagged this buffer for special treatment? */
2499           if (PREDICT_FALSE
2500               ((error0 == 0)
2501                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2502             next0 = hm->next_override;
2503
2504           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2505             {
2506               ip6_hop_by_hop_trace_t *t =
2507                 vlib_add_trace (vm, node, b0, sizeof (*t));
2508               u32 trace_len = (hbh0->length + 1) << 3;
2509               t->next_index = next0;
2510               /* Capture the h-b-h option verbatim */
2511               trace_len =
2512                 trace_len <
2513                 ARRAY_LEN (t->option_data) ? trace_len :
2514                 ARRAY_LEN (t->option_data);
2515               t->trace_len = trace_len;
2516               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2517             }
2518
2519           b0->error = error_node->errors[error0];
2520
2521           /* verify speculative enqueue, maybe switch current next frame */
2522           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2523                                            n_left_to_next, bi0, next0);
2524         }
2525       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2526     }
2527   return frame->n_vectors;
2528 }
2529
2530 /* *INDENT-OFF* */
2531 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2532 {
2533   .function = ip6_hop_by_hop,
2534   .name = "ip6-hop-by-hop",
2535   .sibling_of = "ip6-lookup",
2536   .vector_size = sizeof (u32),
2537   .format_trace = format_ip6_hop_by_hop_trace,
2538   .type = VLIB_NODE_TYPE_INTERNAL,
2539   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2540   .error_strings = ip6_hop_by_hop_error_strings,
2541   .n_next_nodes = 0,
2542 };
2543 /* *INDENT-ON* */
2544
2545 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2546
2547 static clib_error_t *
2548 ip6_hop_by_hop_init (vlib_main_t * vm)
2549 {
2550   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2551   clib_memset (hm->options, 0, sizeof (hm->options));
2552   clib_memset (hm->trace, 0, sizeof (hm->trace));
2553   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2554   return (0);
2555 }
2556
2557 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2558
2559 void
2560 ip6_hbh_set_next_override (uword next)
2561 {
2562   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2563
2564   hm->next_override = next;
2565 }
2566
2567 int
2568 ip6_hbh_register_option (u8 option,
2569                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2570                                       ip6_hop_by_hop_option_t * opt),
2571                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2572 {
2573   ip6_main_t *im = &ip6_main;
2574   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2575
2576   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2577
2578   /* Already registered */
2579   if (hm->options[option])
2580     return (-1);
2581
2582   hm->options[option] = options;
2583   hm->trace[option] = trace;
2584
2585   /* Set global variable */
2586   im->hbh_enabled = 1;
2587
2588   return (0);
2589 }
2590
2591 int
2592 ip6_hbh_unregister_option (u8 option)
2593 {
2594   ip6_main_t *im = &ip6_main;
2595   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2596
2597   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2598
2599   /* Not registered */
2600   if (!hm->options[option])
2601     return (-1);
2602
2603   hm->options[option] = NULL;
2604   hm->trace[option] = NULL;
2605
2606   /* Disable global knob if this was the last option configured */
2607   int i;
2608   bool found = false;
2609   for (i = 0; i < 256; i++)
2610     {
2611       if (hm->options[option])
2612         {
2613           found = true;
2614           break;
2615         }
2616     }
2617   if (!found)
2618     im->hbh_enabled = 0;
2619
2620   return (0);
2621 }
2622
2623 /* Global IP6 main. */
2624 ip6_main_t ip6_main;
2625
2626 static clib_error_t *
2627 ip6_lookup_init (vlib_main_t * vm)
2628 {
2629   ip6_main_t *im = &ip6_main;
2630   clib_error_t *error;
2631   uword i;
2632
2633   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2634     return error;
2635
2636   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2637     {
2638       u32 j, i0, i1;
2639
2640       i0 = i / 32;
2641       i1 = i % 32;
2642
2643       for (j = 0; j < i0; j++)
2644         im->fib_masks[i].as_u32[j] = ~0;
2645
2646       if (i1)
2647         im->fib_masks[i].as_u32[i0] =
2648           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2649     }
2650
2651   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2652
2653   if (im->lookup_table_nbuckets == 0)
2654     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2655
2656   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2657
2658   if (im->lookup_table_size == 0)
2659     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2660
2661   BV (clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2662                          "ip6 FIB fwding table",
2663                          im->lookup_table_nbuckets, im->lookup_table_size);
2664   BV (clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2665                          "ip6 FIB non-fwding table",
2666                          im->lookup_table_nbuckets, im->lookup_table_size);
2667
2668   /* Create FIB with index 0 and table id of 0. */
2669   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2670                                      FIB_SOURCE_DEFAULT_ROUTE);
2671   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2672                                       MFIB_SOURCE_DEFAULT_ROUTE);
2673
2674   {
2675     pg_node_t *pn;
2676     pn = pg_get_node (ip6_lookup_node.index);
2677     pn->unformat_edit = unformat_pg_ip6_header;
2678   }
2679
2680   /* Unless explicitly configured, don't process HBH options */
2681   im->hbh_enabled = 0;
2682
2683   {
2684     icmp6_neighbor_solicitation_header_t p;
2685
2686     clib_memset (&p, 0, sizeof (p));
2687
2688     p.ip.ip_version_traffic_class_and_flow_label =
2689       clib_host_to_net_u32 (0x6 << 28);
2690     p.ip.payload_length =
2691       clib_host_to_net_u16 (sizeof (p) -
2692                             STRUCT_OFFSET_OF
2693                             (icmp6_neighbor_solicitation_header_t, neighbor));
2694     p.ip.protocol = IP_PROTOCOL_ICMP6;
2695     p.ip.hop_limit = 255;
2696     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2697
2698     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2699
2700     p.link_layer_option.header.type =
2701       ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2702     p.link_layer_option.header.n_data_u64s =
2703       sizeof (p.link_layer_option) / sizeof (u64);
2704
2705     vlib_packet_template_init (vm,
2706                                &im->discover_neighbor_packet_template,
2707                                &p, sizeof (p),
2708                                /* alloc chunk size */ 8,
2709                                "ip6 neighbor discovery");
2710   }
2711
2712   return error;
2713 }
2714
2715 VLIB_INIT_FUNCTION (ip6_lookup_init);
2716
2717 void
2718 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t * ip,
2719                                                   u8 * mac)
2720 {
2721   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2722   /* Invert the "u" bit */
2723   ip->as_u8[8] = mac[0] ^ (1 << 1);
2724   ip->as_u8[9] = mac[1];
2725   ip->as_u8[10] = mac[2];
2726   ip->as_u8[11] = 0xFF;
2727   ip->as_u8[12] = 0xFE;
2728   ip->as_u8[13] = mac[3];
2729   ip->as_u8[14] = mac[4];
2730   ip->as_u8[15] = mac[5];
2731 }
2732
2733 void
2734 ip6_ethernet_mac_address_from_link_local_address (u8 * mac,
2735                                                   ip6_address_t * ip)
2736 {
2737   /* Invert the previously inverted "u" bit */
2738   mac[0] = ip->as_u8[8] ^ (1 << 1);
2739   mac[1] = ip->as_u8[9];
2740   mac[2] = ip->as_u8[10];
2741   mac[3] = ip->as_u8[13];
2742   mac[4] = ip->as_u8[14];
2743   mac[5] = ip->as_u8[15];
2744 }
2745
2746 static clib_error_t *
2747 test_ip6_link_command_fn (vlib_main_t * vm,
2748                           unformat_input_t * input, vlib_cli_command_t * cmd)
2749 {
2750   u8 mac[6];
2751   ip6_address_t _a, *a = &_a;
2752
2753   if (unformat (input, "%U", unformat_ethernet_address, mac))
2754     {
2755       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2756       vlib_cli_output (vm, "Link local address: %U", format_ip6_address, a);
2757       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2758       vlib_cli_output (vm, "Original MAC address: %U",
2759                        format_ethernet_address, mac);
2760     }
2761
2762   return 0;
2763 }
2764
2765 /*?
2766  * This command converts the given MAC Address into an IPv6 link-local
2767  * address.
2768  *
2769  * @cliexpar
2770  * Example of how to create an IPv6 link-local address:
2771  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2772  * Link local address: fe80::14d9:e0ff:fe91:7986
2773  * Original MAC address: 16:d9:e0:91:79:86
2774  * @cliexend
2775 ?*/
2776 /* *INDENT-OFF* */
2777 VLIB_CLI_COMMAND (test_link_command, static) =
2778 {
2779   .path = "test ip6 link",
2780   .function = test_ip6_link_command_fn,
2781   .short_help = "test ip6 link <mac-address>",
2782 };
2783 /* *INDENT-ON* */
2784
2785 int
2786 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2787 {
2788   u32 fib_index;
2789
2790   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2791
2792   if (~0 == fib_index)
2793     return VNET_API_ERROR_NO_SUCH_FIB;
2794
2795   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2796                                   flow_hash_config);
2797
2798   return 0;
2799 }
2800
2801 static clib_error_t *
2802 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2803                               unformat_input_t * input,
2804                               vlib_cli_command_t * cmd)
2805 {
2806   int matched = 0;
2807   u32 table_id = 0;
2808   u32 flow_hash_config = 0;
2809   int rv;
2810
2811   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2812     {
2813       if (unformat (input, "table %d", &table_id))
2814         matched = 1;
2815 #define _(a,v) \
2816     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2817       foreach_flow_hash_bit
2818 #undef _
2819         else
2820         break;
2821     }
2822
2823   if (matched == 0)
2824     return clib_error_return (0, "unknown input `%U'",
2825                               format_unformat_error, input);
2826
2827   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2828   switch (rv)
2829     {
2830     case 0:
2831       break;
2832
2833     case -1:
2834       return clib_error_return (0, "no such FIB table %d", table_id);
2835
2836     default:
2837       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2838       break;
2839     }
2840
2841   return 0;
2842 }
2843
2844 /*?
2845  * Configure the set of IPv6 fields used by the flow hash.
2846  *
2847  * @cliexpar
2848  * @parblock
2849  * Example of how to set the flow hash on a given table:
2850  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2851  *
2852  * Example of display the configured flow hash:
2853  * @cliexstart{show ip6 fib}
2854  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2855  * @::/0
2856  *   unicast-ip6-chain
2857  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2858  *     [0] [@0]: dpo-drop ip6
2859  * fe80::/10
2860  *   unicast-ip6-chain
2861  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2862  *     [0] [@2]: dpo-receive
2863  * ff02::1/128
2864  *   unicast-ip6-chain
2865  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2866  *     [0] [@2]: dpo-receive
2867  * ff02::2/128
2868  *   unicast-ip6-chain
2869  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2870  *     [0] [@2]: dpo-receive
2871  * ff02::16/128
2872  *   unicast-ip6-chain
2873  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2874  *     [0] [@2]: dpo-receive
2875  * ff02::1:ff00:0/104
2876  *   unicast-ip6-chain
2877  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2878  *     [0] [@2]: dpo-receive
2879  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2880  * @::/0
2881  *   unicast-ip6-chain
2882  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2883  *     [0] [@0]: dpo-drop ip6
2884  * @::a:1:1:0:4/126
2885  *   unicast-ip6-chain
2886  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2887  *     [0] [@4]: ipv6-glean: af_packet0
2888  * @::a:1:1:0:7/128
2889  *   unicast-ip6-chain
2890  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2891  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2892  * fe80::/10
2893  *   unicast-ip6-chain
2894  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2895  *     [0] [@2]: dpo-receive
2896  * fe80::fe:3eff:fe3e:9222/128
2897  *   unicast-ip6-chain
2898  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2899  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2900  * ff02::1/128
2901  *   unicast-ip6-chain
2902  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2903  *     [0] [@2]: dpo-receive
2904  * ff02::2/128
2905  *   unicast-ip6-chain
2906  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2907  *     [0] [@2]: dpo-receive
2908  * ff02::16/128
2909  *   unicast-ip6-chain
2910  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2911  *     [0] [@2]: dpo-receive
2912  * ff02::1:ff00:0/104
2913  *   unicast-ip6-chain
2914  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2915  *     [0] [@2]: dpo-receive
2916  * @cliexend
2917  * @endparblock
2918 ?*/
2919 /* *INDENT-OFF* */
2920 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
2921 {
2922   .path = "set ip6 flow-hash",
2923   .short_help =
2924   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2925   .function = set_ip6_flow_hash_command_fn,
2926 };
2927 /* *INDENT-ON* */
2928
2929 static clib_error_t *
2930 show_ip6_local_command_fn (vlib_main_t * vm,
2931                            unformat_input_t * input, vlib_cli_command_t * cmd)
2932 {
2933   ip6_main_t *im = &ip6_main;
2934   ip_lookup_main_t *lm = &im->lookup_main;
2935   int i;
2936
2937   vlib_cli_output (vm, "Protocols handled by ip6_local");
2938   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
2939     {
2940       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2941         {
2942
2943           u32 node_index = vlib_get_node (vm,
2944                                           ip6_local_node.index)->
2945             next_nodes[lm->local_next_by_ip_protocol[i]];
2946           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
2947                            node_index);
2948         }
2949     }
2950   return 0;
2951 }
2952
2953
2954
2955 /*?
2956  * Display the set of protocols handled by the local IPv6 stack.
2957  *
2958  * @cliexpar
2959  * Example of how to display local protocol table:
2960  * @cliexstart{show ip6 local}
2961  * Protocols handled by ip6_local
2962  * 17
2963  * 43
2964  * 58
2965  * 115
2966  * @cliexend
2967 ?*/
2968 /* *INDENT-OFF* */
2969 VLIB_CLI_COMMAND (show_ip6_local, static) =
2970 {
2971   .path = "show ip6 local",
2972   .function = show_ip6_local_command_fn,
2973   .short_help = "show ip6 local",
2974 };
2975 /* *INDENT-ON* */
2976
2977 int
2978 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2979                              u32 table_index)
2980 {
2981   vnet_main_t *vnm = vnet_get_main ();
2982   vnet_interface_main_t *im = &vnm->interface_main;
2983   ip6_main_t *ipm = &ip6_main;
2984   ip_lookup_main_t *lm = &ipm->lookup_main;
2985   vnet_classify_main_t *cm = &vnet_classify_main;
2986   ip6_address_t *if_addr;
2987
2988   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2989     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2990
2991   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2992     return VNET_API_ERROR_NO_SUCH_ENTRY;
2993
2994   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2995   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2996
2997   if_addr = ip6_interface_first_address (ipm, sw_if_index);
2998
2999   if (NULL != if_addr)
3000     {
3001       fib_prefix_t pfx = {
3002         .fp_len = 128,
3003         .fp_proto = FIB_PROTOCOL_IP6,
3004         .fp_addr.ip6 = *if_addr,
3005       };
3006       u32 fib_index;
3007
3008       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3009                                                        sw_if_index);
3010
3011
3012       if (table_index != (u32) ~ 0)
3013         {
3014           dpo_id_t dpo = DPO_INVALID;
3015
3016           dpo_set (&dpo,
3017                    DPO_CLASSIFY,
3018                    DPO_PROTO_IP6,
3019                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3020
3021           fib_table_entry_special_dpo_add (fib_index,
3022                                            &pfx,
3023                                            FIB_SOURCE_CLASSIFY,
3024                                            FIB_ENTRY_FLAG_NONE, &dpo);
3025           dpo_reset (&dpo);
3026         }
3027       else
3028         {
3029           fib_table_entry_special_remove (fib_index,
3030                                           &pfx, FIB_SOURCE_CLASSIFY);
3031         }
3032     }
3033
3034   return 0;
3035 }
3036
3037 static clib_error_t *
3038 set_ip6_classify_command_fn (vlib_main_t * vm,
3039                              unformat_input_t * input,
3040                              vlib_cli_command_t * cmd)
3041 {
3042   u32 table_index = ~0;
3043   int table_index_set = 0;
3044   u32 sw_if_index = ~0;
3045   int rv;
3046
3047   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3048     {
3049       if (unformat (input, "table-index %d", &table_index))
3050         table_index_set = 1;
3051       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3052                          vnet_get_main (), &sw_if_index))
3053         ;
3054       else
3055         break;
3056     }
3057
3058   if (table_index_set == 0)
3059     return clib_error_return (0, "classify table-index must be specified");
3060
3061   if (sw_if_index == ~0)
3062     return clib_error_return (0, "interface / subif must be specified");
3063
3064   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3065
3066   switch (rv)
3067     {
3068     case 0:
3069       break;
3070
3071     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3072       return clib_error_return (0, "No such interface");
3073
3074     case VNET_API_ERROR_NO_SUCH_ENTRY:
3075       return clib_error_return (0, "No such classifier table");
3076     }
3077   return 0;
3078 }
3079
3080 /*?
3081  * Assign a classification table to an interface. The classification
3082  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3083  * commands. Once the table is create, use this command to filter packets
3084  * on an interface.
3085  *
3086  * @cliexpar
3087  * Example of how to assign a classification table to an interface:
3088  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3089 ?*/
3090 /* *INDENT-OFF* */
3091 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3092 {
3093   .path = "set ip6 classify",
3094   .short_help =
3095   "set ip6 classify intfc <interface> table-index <classify-idx>",
3096   .function = set_ip6_classify_command_fn,
3097 };
3098 /* *INDENT-ON* */
3099
3100 static clib_error_t *
3101 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3102 {
3103   ip6_main_t *im = &ip6_main;
3104   uword heapsize = 0;
3105   u32 tmp;
3106   u32 nbuckets = 0;
3107
3108   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3109     {
3110       if (unformat (input, "hash-buckets %d", &tmp))
3111         nbuckets = tmp;
3112       else if (unformat (input, "heap-size %U",
3113                          unformat_memory_size, &heapsize))
3114         ;
3115       else
3116         return clib_error_return (0, "unknown input '%U'",
3117                                   format_unformat_error, input);
3118     }
3119
3120   im->lookup_table_nbuckets = nbuckets;
3121   im->lookup_table_size = heapsize;
3122
3123   return 0;
3124 }
3125
3126 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3127
3128 /*
3129  * fd.io coding-style-patch-verification: ON
3130  *
3131  * Local Variables:
3132  * eval: (c-set-style "gnu")
3133  * End:
3134  */