src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_neighbor.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52
  53 #ifndef CLIB_MARCH_VARIANT
  54 #include <vppinfra/bihash_template.c>
  55 #endif
  56 #include <vnet/ip/ip6_forward.h>
  57 #include <vnet/interface_output.h>
  58
  59 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  60 #define OI_DECAP   0x80000000
  61
  62 static void
  63 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
  64                           ip6_main_t * im, u32 fib_index,
  65                           ip_interface_address_t * a)
  66 {
  67   ip_lookup_main_t *lm = &im->lookup_main;
  68   ip6_address_t *address = ip_interface_address_get_address (lm, a);
  69   fib_prefix_t pfx = {
  70     .fp_len = a->address_length,
  71     .fp_proto = FIB_PROTOCOL_IP6,
  72     .fp_addr.ip6 = *address,
  73   };
  74
  75   if (a->address_length < 128)
  76     {
  77       fib_table_entry_update_one_path (fib_index,
  78                                        &pfx,
  79                                        FIB_SOURCE_INTERFACE,
  80                                        (FIB_ENTRY_FLAG_CONNECTED |
  81                                         FIB_ENTRY_FLAG_ATTACHED),
  82                                        DPO_PROTO_IP6,
  83                                        /* No next-hop address */
  84                                        NULL, sw_if_index,
  85                                        /* invalid FIB index */
  86                                        ~0, 1,
  87                                        /* no label stack */
  88                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
  89     }
  90
  91   pfx.fp_len = 128;
  92   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
  93     {
  94       u32 classify_table_index =
  95         lm->classify_table_index_by_sw_if_index[sw_if_index];
  96       if (classify_table_index != (u32) ~ 0)
  97         {
  98           dpo_id_t dpo = DPO_INVALID;
  99
 100           dpo_set (&dpo,
 101                    DPO_CLASSIFY,
 102                    DPO_PROTO_IP6,
 103                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 104
 105           fib_table_entry_special_dpo_add (fib_index,
 106                                            &pfx,
 107                                            FIB_SOURCE_CLASSIFY,
 108                                            FIB_ENTRY_FLAG_NONE, &dpo);
 109           dpo_reset (&dpo);
 110         }
 111     }
 112
 113   fib_table_entry_update_one_path (fib_index, &pfx,
 114                                    FIB_SOURCE_INTERFACE,
 115                                    (FIB_ENTRY_FLAG_CONNECTED |
 116                                     FIB_ENTRY_FLAG_LOCAL),
 117                                    DPO_PROTO_IP6,
 118                                    &pfx.fp_addr,
 119                                    sw_if_index, ~0,
 120                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 121 }
 122
 123 static void
 124 ip6_del_interface_routes (ip6_main_t * im,
 125                           u32 fib_index,
 126                           ip6_address_t * address, u32 address_length)
 127 {
 128   fib_prefix_t pfx = {
 129     .fp_len = address_length,
 130     .fp_proto = FIB_PROTOCOL_IP6,
 131     .fp_addr.ip6 = *address,
 132   };
 133
 134   if (pfx.fp_len < 128)
 135     {
 136       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 137
 138     }
 139
 140   pfx.fp_len = 128;
 141   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 142 }
 143
 144 #ifndef CLIB_MARCH_VARIANT
 145 void
 146 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 147 {
 148   ip6_main_t *im = &ip6_main;
 149
 150   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 151
 152   /*
 153    * enable/disable only on the 1<->0 transition
 154    */
 155   if (is_enable)
 156     {
 157       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 158         return;
 159     }
 160   else
 161     {
 162       /* The ref count is 0 when an address is removed from an interface that has
 163        * no address - this is not a ciritical error */
 164       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 165           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 166         return;
 167     }
 168
 169   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 170                                !is_enable, 0, 0);
 171
 172   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 173                                sw_if_index, !is_enable, 0, 0);
 174 }
 175
 176 /* get first interface address */
 177 ip6_address_t *
 178 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 179 {
 180   ip_lookup_main_t *lm = &im->lookup_main;
 181   ip_interface_address_t *ia = 0;
 182   ip6_address_t *result = 0;
 183
 184   /* *INDENT-OFF* */
 185   foreach_ip_interface_address (lm, ia, sw_if_index,
 186                                 1 /* honor unnumbered */,
 187   ({
 188     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 189     result = a;
 190     break;
 191   }));
 192   /* *INDENT-ON* */
 193   return result;
 194 }
 195
 196 clib_error_t *
 197 ip6_add_del_interface_address (vlib_main_t * vm,
 198                                u32 sw_if_index,
 199                                ip6_address_t * address,
 200                                u32 address_length, u32 is_del)
 201 {
 202   vnet_main_t *vnm = vnet_get_main ();
 203   ip6_main_t *im = &ip6_main;
 204   ip_lookup_main_t *lm = &im->lookup_main;
 205   clib_error_t *error;
 206   u32 if_address_index;
 207   ip6_address_fib_t ip6_af, *addr_fib = 0;
 208   ip6_address_t ll_addr;
 209
 210   /* local0 interface doesn't support IP addressing */
 211   if (sw_if_index == 0)
 212     {
 213       return
 214         clib_error_create ("local0 interface doesn't support IP addressing");
 215     }
 216
 217   if (ip6_address_is_link_local_unicast (address))
 218     {
 219       if (address_length != 128)
 220         {
 221           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 222           return
 223             clib_error_create
 224             ("prefix length of link-local address must be 128");
 225         }
 226       if (!is_del)
 227         {
 228           return ip6_neighbor_set_link_local_address (vm, sw_if_index,
 229                                                       address);
 230         }
 231       else
 232         {
 233           ll_addr = ip6_neighbor_get_link_local_address (sw_if_index);
 234           if (ip6_address_is_equal (&ll_addr, address))
 235             {
 236               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 237               return clib_error_create ("address not deletable");
 238             }
 239           else
 240             {
 241               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 242               return clib_error_create ("address not found");
 243             }
 244         }
 245     }
 246
 247   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 248   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 249
 250   ip6_addr_fib_init (&ip6_af, address,
 251                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 252   vec_add1 (addr_fib, ip6_af);
 253
 254   /* *INDENT-OFF* */
 255   if (!is_del)
 256     {
 257       /* When adding an address check that it does not conflict
 258          with an existing address on any interface in this table. */
 259       ip_interface_address_t *ia;
 260       vnet_sw_interface_t *sif;
 261
 262       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 263       ({
 264           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 265               im->fib_index_by_sw_if_index[sif->sw_if_index])
 266             {
 267               foreach_ip_interface_address
 268                 (&im->lookup_main, ia, sif->sw_if_index,
 269                  0 /* honor unnumbered */ ,
 270                  ({
 271                    ip6_address_t * x =
 272                      ip_interface_address_get_address
 273                      (&im->lookup_main, ia);
 274                    if (ip6_destination_matches_route
 275                        (im, address, x, ia->address_length) ||
 276                        ip6_destination_matches_route (im,
 277                                                       x,
 278                                                       address,
 279                                                       address_length))
 280                      {
 281                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 282                        return
 283                          clib_error_create
 284                          ("failed to add %U which conflicts with %U for interface %U",
 285                           format_ip6_address_and_length, address,
 286                           address_length,
 287                           format_ip6_address_and_length, x,
 288                           ia->address_length,
 289                           format_vnet_sw_if_index_name, vnm,
 290                           sif->sw_if_index);
 291                      }
 292                  }));
 293             }
 294       }));
 295     }
 296   /* *INDENT-ON* */
 297
 298   {
 299     uword elts_before = pool_elts (lm->if_address_pool);
 300
 301     error = ip_interface_address_add_del
 302       (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 303     if (error)
 304       goto done;
 305
 306     /* Pool did not grow: add duplicate address. */
 307     if (elts_before == pool_elts (lm->if_address_pool))
 308       goto done;
 309   }
 310
 311   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 312
 313   if (is_del)
 314     ip6_del_interface_routes (im, ip6_af.fib_index, address, address_length);
 315   else
 316     ip6_add_interface_routes (vnm, sw_if_index,
 317                               im, ip6_af.fib_index,
 318                               pool_elt_at_index (lm->if_address_pool,
 319                                                  if_address_index));
 320
 321   {
 322     ip6_add_del_interface_address_callback_t *cb;
 323     vec_foreach (cb, im->add_del_interface_address_callbacks)
 324       cb->function (im, cb->function_opaque, sw_if_index,
 325                     address, address_length, if_address_index, is_del);
 326   }
 327
 328 done:
 329   vec_free (addr_fib);
 330   return error;
 331 }
 332
 333 #endif
 334
 335 static clib_error_t *
 336 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 337 {
 338   ip6_main_t *im = &ip6_main;
 339   ip_interface_address_t *ia;
 340   ip6_address_t *a;
 341   u32 is_admin_up, fib_index;
 342
 343   /* Fill in lookup tables with default table (0). */
 344   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 345
 346   vec_validate_init_empty (im->
 347                            lookup_main.if_address_pool_index_by_sw_if_index,
 348                            sw_if_index, ~0);
 349
 350   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 351
 352   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 353
 354   /* *INDENT-OFF* */
 355   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 356                                 0 /* honor unnumbered */,
 357   ({
 358     a = ip_interface_address_get_address (&im->lookup_main, ia);
 359     if (is_admin_up)
 360       ip6_add_interface_routes (vnm, sw_if_index,
 361                                 im, fib_index,
 362                                 ia);
 363     else
 364       ip6_del_interface_routes (im, fib_index,
 365                                 a, ia->address_length);
 366   }));
 367   /* *INDENT-ON* */
 368
 369   return 0;
 370 }
 371
 372 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 373
 374 /* Built-in ip6 unicast rx feature path definition */
 375 /* *INDENT-OFF* */
 376 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 377 {
 378   .arc_name  = "ip6-unicast",
 379   .start_nodes = VNET_FEATURES ("ip6-input"),
 380   .last_in_arc = "ip6-lookup",
 381   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 382 };
 383
 384 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 385 {
 386   .arc_name = "ip6-unicast",
 387   .node_name = "ip6-flow-classify",
 388   .runs_before = VNET_FEATURES ("ip6-inacl"),
 389 };
 390
 391 VNET_FEATURE_INIT (ip6_inacl, static) =
 392 {
 393   .arc_name = "ip6-unicast",
 394   .node_name = "ip6-inacl",
 395   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 396 };
 397
 398 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 399 {
 400   .arc_name = "ip6-unicast",
 401   .node_name = "ip6-policer-classify",
 402   .runs_before = VNET_FEATURES ("ipsec6-input-feature"),
 403 };
 404
 405 VNET_FEATURE_INIT (ip6_ipsec, static) =
 406 {
 407   .arc_name = "ip6-unicast",
 408   .node_name = "ipsec6-input-feature",
 409   .runs_before = VNET_FEATURES ("l2tp-decap"),
 410 };
 411
 412 VNET_FEATURE_INIT (ip6_l2tp, static) =
 413 {
 414   .arc_name = "ip6-unicast",
 415   .node_name = "l2tp-decap",
 416   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 417 };
 418
 419 VNET_FEATURE_INIT (ip6_vpath, static) =
 420 {
 421   .arc_name = "ip6-unicast",
 422   .node_name = "vpath-input-ip6",
 423   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 424 };
 425
 426 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 427 {
 428   .arc_name = "ip6-unicast",
 429   .node_name = "ip6-vxlan-bypass",
 430   .runs_before = VNET_FEATURES ("ip6-lookup"),
 431 };
 432
 433 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 434 {
 435   .arc_name = "ip6-unicast",
 436   .node_name = "ip6-not-enabled",
 437   .runs_before = VNET_FEATURES ("ip6-lookup"),
 438 };
 439
 440 VNET_FEATURE_INIT (ip6_lookup, static) =
 441 {
 442   .arc_name = "ip6-unicast",
 443   .node_name = "ip6-lookup",
 444   .runs_before = 0,  /*last feature*/
 445 };
 446
 447 /* Built-in ip6 multicast rx feature path definition (none now) */
 448 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 449 {
 450   .arc_name  = "ip6-multicast",
 451   .start_nodes = VNET_FEATURES ("ip6-input"),
 452   .last_in_arc = "ip6-mfib-forward-lookup",
 453   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 454 };
 455
 456 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 457   .arc_name = "ip6-multicast",
 458   .node_name = "vpath-input-ip6",
 459   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 460 };
 461
 462 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 463   .arc_name = "ip6-multicast",
 464   .node_name = "ip6-not-enabled",
 465   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 466 };
 467
 468 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 469   .arc_name = "ip6-multicast",
 470   .node_name = "ip6-mfib-forward-lookup",
 471   .runs_before = 0, /* last feature */
 472 };
 473
 474 /* Built-in ip4 tx feature path definition */
 475 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 476 {
 477   .arc_name  = "ip6-output",
 478   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 479   .last_in_arc = "interface-output",
 480   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 481 };
 482
 483 VNET_FEATURE_INIT (ip6_outacl, static) = {
 484   .arc_name = "ip6-output",
 485   .node_name = "ip6-outacl",
 486   .runs_before = VNET_FEATURES ("ipsec6-output-feature"),
 487 };
 488
 489 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 490   .arc_name = "ip6-output",
 491   .node_name = "ipsec6-output-feature",
 492   .runs_before = VNET_FEATURES ("interface-output"),
 493 };
 494
 495 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 496   .arc_name = "ip6-output",
 497   .node_name = "interface-output",
 498   .runs_before = 0, /* not before any other features */
 499 };
 500 /* *INDENT-ON* */
 501
 502 static clib_error_t *
 503 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 504 {
 505   ip6_main_t *im = &ip6_main;
 506
 507   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 508   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 509
 510   if (!is_add)
 511     {
 512       /* Ensure that IPv6 is disabled */
 513       ip6_main_t *im6 = &ip6_main;
 514       ip_lookup_main_t *lm6 = &im6->lookup_main;
 515       ip_interface_address_t *ia = 0;
 516       ip6_address_t *address;
 517       vlib_main_t *vm = vlib_get_main ();
 518
 519       ip6_neighbor_sw_interface_add_del (vnm, sw_if_index, 0 /* is_add */ );
 520       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 521       /* *INDENT-OFF* */
 522       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 523       ({
 524         address = ip_interface_address_get_address (lm6, ia);
 525         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 526       }));
 527       /* *INDENT-ON* */
 528       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 529     }
 530
 531   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 532                                is_add, 0, 0);
 533
 534   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 535                                sw_if_index, is_add, 0, 0);
 536
 537   return /* no error */ 0;
 538 }
 539
 540 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 541
 542 VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 543                                 vlib_node_runtime_t * node,
 544                                 vlib_frame_t * frame)
 545 {
 546   return ip6_lookup_inline (vm, node, frame);
 547 }
 548
 549 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 550
 551 /* *INDENT-OFF* */
 552 VLIB_REGISTER_NODE (ip6_lookup_node) =
 553 {
 554   .name = "ip6-lookup",
 555   .vector_size = sizeof (u32),
 556   .format_trace = format_ip6_lookup_trace,
 557   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 558   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 559 };
 560 /* *INDENT-ON* */
 561
 562 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
 563                                       vlib_node_runtime_t * node,
 564                                       vlib_frame_t * frame)
 565 {
 566   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 567   u32 n_left_from, n_left_to_next, *from, *to_next;
 568   ip_lookup_next_t next;
 569   u32 thread_index = vm->thread_index;
 570   ip6_main_t *im = &ip6_main;
 571
 572   from = vlib_frame_vector_args (frame);
 573   n_left_from = frame->n_vectors;
 574   next = node->cached_next_index;
 575
 576   while (n_left_from > 0)
 577     {
 578       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 579
 580
 581       while (n_left_from >= 4 && n_left_to_next >= 2)
 582         {
 583           ip_lookup_next_t next0, next1;
 584           const load_balance_t *lb0, *lb1;
 585           vlib_buffer_t *p0, *p1;
 586           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 587           const ip6_header_t *ip0, *ip1;
 588           const dpo_id_t *dpo0, *dpo1;
 589
 590           /* Prefetch next iteration. */
 591           {
 592             vlib_buffer_t *p2, *p3;
 593
 594             p2 = vlib_get_buffer (vm, from[2]);
 595             p3 = vlib_get_buffer (vm, from[3]);
 596
 597             vlib_prefetch_buffer_header (p2, STORE);
 598             vlib_prefetch_buffer_header (p3, STORE);
 599
 600             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 601             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 602           }
 603
 604           pi0 = to_next[0] = from[0];
 605           pi1 = to_next[1] = from[1];
 606
 607           from += 2;
 608           n_left_from -= 2;
 609           to_next += 2;
 610           n_left_to_next -= 2;
 611
 612           p0 = vlib_get_buffer (vm, pi0);
 613           p1 = vlib_get_buffer (vm, pi1);
 614
 615           ip0 = vlib_buffer_get_current (p0);
 616           ip1 = vlib_buffer_get_current (p1);
 617           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 618           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 619
 620           lb0 = load_balance_get (lbi0);
 621           lb1 = load_balance_get (lbi1);
 622
 623           /*
 624            * this node is for via FIBs we can re-use the hash value from the
 625            * to node if present.
 626            * We don't want to use the same hash value at each level in the recursion
 627            * graph as that would lead to polarisation
 628            */
 629           hc0 = hc1 = 0;
 630
 631           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 632             {
 633               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 634                 {
 635                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 636                     vnet_buffer (p0)->ip.flow_hash >> 1;
 637                 }
 638               else
 639                 {
 640                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 641                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 642                 }
 643               dpo0 =
 644                 load_balance_get_fwd_bucket (lb0,
 645                                              (hc0 &
 646                                               lb0->lb_n_buckets_minus_1));
 647             }
 648           else
 649             {
 650               dpo0 = load_balance_get_bucket_i (lb0, 0);
 651             }
 652           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 653             {
 654               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 655                 {
 656                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 657                     vnet_buffer (p1)->ip.flow_hash >> 1;
 658                 }
 659               else
 660                 {
 661                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 662                     ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 663                 }
 664               dpo1 =
 665                 load_balance_get_fwd_bucket (lb1,
 666                                              (hc1 &
 667                                               lb1->lb_n_buckets_minus_1));
 668             }
 669           else
 670             {
 671               dpo1 = load_balance_get_bucket_i (lb1, 0);
 672             }
 673
 674           next0 = dpo0->dpoi_next_node;
 675           next1 = dpo1->dpoi_next_node;
 676
 677           /* Only process the HBH Option Header if explicitly configured to do so */
 678           if (PREDICT_FALSE
 679               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 680             {
 681               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 682                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 683             }
 684           /* Only process the HBH Option Header if explicitly configured to do so */
 685           if (PREDICT_FALSE
 686               (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 687             {
 688               next1 = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 689                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 690             }
 691
 692           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 693           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 694
 695           vlib_increment_combined_counter
 696             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 697           vlib_increment_combined_counter
 698             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 699
 700           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 701                                            to_next, n_left_to_next,
 702                                            pi0, pi1, next0, next1);
 703         }
 704
 705       while (n_left_from > 0 && n_left_to_next > 0)
 706         {
 707           ip_lookup_next_t next0;
 708           const load_balance_t *lb0;
 709           vlib_buffer_t *p0;
 710           u32 pi0, lbi0, hc0;
 711           const ip6_header_t *ip0;
 712           const dpo_id_t *dpo0;
 713
 714           pi0 = from[0];
 715           to_next[0] = pi0;
 716           from += 1;
 717           to_next += 1;
 718           n_left_to_next -= 1;
 719           n_left_from -= 1;
 720
 721           p0 = vlib_get_buffer (vm, pi0);
 722
 723           ip0 = vlib_buffer_get_current (p0);
 724           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 725
 726           lb0 = load_balance_get (lbi0);
 727
 728           hc0 = 0;
 729           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 730             {
 731               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 732                 {
 733                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 734                     vnet_buffer (p0)->ip.flow_hash >> 1;
 735                 }
 736               else
 737                 {
 738                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 739                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 740                 }
 741               dpo0 =
 742                 load_balance_get_fwd_bucket (lb0,
 743                                              (hc0 &
 744                                               lb0->lb_n_buckets_minus_1));
 745             }
 746           else
 747             {
 748               dpo0 = load_balance_get_bucket_i (lb0, 0);
 749             }
 750
 751           next0 = dpo0->dpoi_next_node;
 752           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 753
 754           /* Only process the HBH Option Header if explicitly configured to do so */
 755           if (PREDICT_FALSE
 756               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 757             {
 758               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 759                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 760             }
 761
 762           vlib_increment_combined_counter
 763             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 764
 765           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 766                                            to_next, n_left_to_next,
 767                                            pi0, next0);
 768         }
 769
 770       vlib_put_next_frame (vm, node, next, n_left_to_next);
 771     }
 772
 773   if (node->flags & VLIB_NODE_FLAG_TRACE)
 774     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 775
 776   return frame->n_vectors;
 777 }
 778
 779 /* *INDENT-OFF* */
 780 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 781 {
 782   .name = "ip6-load-balance",
 783   .vector_size = sizeof (u32),
 784   .sibling_of = "ip6-lookup",
 785   .format_trace = format_ip6_lookup_trace,
 786 };
 787 /* *INDENT-ON* */
 788
 789 typedef struct
 790 {
 791   /* Adjacency taken. */
 792   u32 adj_index;
 793   u32 flow_hash;
 794   u32 fib_index;
 795
 796   /* Packet data, possibly *after* rewrite. */
 797   u8 packet_data[128 - 1 * sizeof (u32)];
 798 }
 799 ip6_forward_next_trace_t;
 800
 801 #ifndef CLIB_MARCH_VARIANT
 802 u8 *
 803 format_ip6_forward_next_trace (u8 * s, va_list * args)
 804 {
 805   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 806   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 807   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 808   u32 indent = format_get_indent (s);
 809
 810   s = format (s, "%U%U",
 811               format_white_space, indent,
 812               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 813   return s;
 814 }
 815 #endif
 816
 817 static u8 *
 818 format_ip6_lookup_trace (u8 * s, va_list * args)
 819 {
 820   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 821   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 822   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 823   u32 indent = format_get_indent (s);
 824
 825   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 826               t->fib_index, t->adj_index, t->flow_hash);
 827   s = format (s, "\n%U%U",
 828               format_white_space, indent,
 829               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 830   return s;
 831 }
 832
 833
 834 static u8 *
 835 format_ip6_rewrite_trace (u8 * s, va_list * args)
 836 {
 837   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 838   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 839   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 840   u32 indent = format_get_indent (s);
 841
 842   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 843               t->fib_index, t->adj_index, format_ip_adjacency,
 844               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 845   s = format (s, "\n%U%U",
 846               format_white_space, indent,
 847               format_ip_adjacency_packet_data,
 848               t->adj_index, t->packet_data, sizeof (t->packet_data));
 849   return s;
 850 }
 851
 852 /* Common trace function for all ip6-forward next nodes. */
 853 #ifndef CLIB_MARCH_VARIANT
 854 void
 855 ip6_forward_next_trace (vlib_main_t * vm,
 856                         vlib_node_runtime_t * node,
 857                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 858 {
 859   u32 *from, n_left;
 860   ip6_main_t *im = &ip6_main;
 861
 862   n_left = frame->n_vectors;
 863   from = vlib_frame_vector_args (frame);
 864
 865   while (n_left >= 4)
 866     {
 867       u32 bi0, bi1;
 868       vlib_buffer_t *b0, *b1;
 869       ip6_forward_next_trace_t *t0, *t1;
 870
 871       /* Prefetch next iteration. */
 872       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 873       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 874
 875       bi0 = from[0];
 876       bi1 = from[1];
 877
 878       b0 = vlib_get_buffer (vm, bi0);
 879       b1 = vlib_get_buffer (vm, bi1);
 880
 881       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 882         {
 883           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 884           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 885           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 886           t0->fib_index =
 887             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 888              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 889             vec_elt (im->fib_index_by_sw_if_index,
 890                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 891
 892           clib_memcpy_fast (t0->packet_data,
 893                             vlib_buffer_get_current (b0),
 894                             sizeof (t0->packet_data));
 895         }
 896       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 897         {
 898           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 899           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 900           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 901           t1->fib_index =
 902             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
 903              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
 904             vec_elt (im->fib_index_by_sw_if_index,
 905                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
 906
 907           clib_memcpy_fast (t1->packet_data,
 908                             vlib_buffer_get_current (b1),
 909                             sizeof (t1->packet_data));
 910         }
 911       from += 2;
 912       n_left -= 2;
 913     }
 914
 915   while (n_left >= 1)
 916     {
 917       u32 bi0;
 918       vlib_buffer_t *b0;
 919       ip6_forward_next_trace_t *t0;
 920
 921       bi0 = from[0];
 922
 923       b0 = vlib_get_buffer (vm, bi0);
 924
 925       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 926         {
 927           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 928           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 929           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 930           t0->fib_index =
 931             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 932              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 933             vec_elt (im->fib_index_by_sw_if_index,
 934                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 935
 936           clib_memcpy_fast (t0->packet_data,
 937                             vlib_buffer_get_current (b0),
 938                             sizeof (t0->packet_data));
 939         }
 940       from += 1;
 941       n_left -= 1;
 942     }
 943 }
 944
 945 /* Compute TCP/UDP/ICMP6 checksum in software. */
 946 u16
 947 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
 948                                    ip6_header_t * ip0, int *bogus_lengthp)
 949 {
 950   ip_csum_t sum0;
 951   u16 sum16, payload_length_host_byte_order;
 952   u32 i, n_this_buffer, n_bytes_left;
 953   u32 headers_size = sizeof (ip0[0]);
 954   void *data_this_buffer;
 955
 956   ASSERT (bogus_lengthp);
 957   *bogus_lengthp = 0;
 958
 959   /* Initialize checksum with ip header. */
 960   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
 961   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
 962   data_this_buffer = (void *) (ip0 + 1);
 963
 964   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
 965     {
 966       sum0 = ip_csum_with_carry (sum0,
 967                                  clib_mem_unaligned (&ip0->
 968                                                      src_address.as_uword[i],
 969                                                      uword));
 970       sum0 =
 971         ip_csum_with_carry (sum0,
 972                             clib_mem_unaligned (&ip0->dst_address.as_uword[i],
 973                                                 uword));
 974     }
 975
 976   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
 977    * or UDP-Ping packets */
 978   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 979     {
 980       u32 skip_bytes;
 981       ip6_hop_by_hop_ext_t *ext_hdr =
 982         (ip6_hop_by_hop_ext_t *) data_this_buffer;
 983
 984       /* validate really icmp6 next */
 985       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
 986               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
 987
 988       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
 989       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
 990
 991       payload_length_host_byte_order -= skip_bytes;
 992       headers_size += skip_bytes;
 993     }
 994
 995   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
 996   if (p0 && n_this_buffer + headers_size > p0->current_length)
 997     n_this_buffer =
 998       p0->current_length >
 999       headers_size ? p0->current_length - headers_size : 0;
1000   while (1)
1001     {
1002       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1003       n_bytes_left -= n_this_buffer;
1004       if (n_bytes_left == 0)
1005         break;
1006
1007       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1008         {
1009           *bogus_lengthp = 1;
1010           return 0xfefe;
1011         }
1012       p0 = vlib_get_buffer (vm, p0->next_buffer);
1013       data_this_buffer = vlib_buffer_get_current (p0);
1014       n_this_buffer = clib_min (p0->current_length, n_bytes_left);
1015     }
1016
1017   sum16 = ~ip_csum_fold (sum0);
1018
1019   return sum16;
1020 }
1021
1022 u32
1023 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1024 {
1025   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1026   udp_header_t *udp0;
1027   u16 sum16;
1028   int bogus_length;
1029
1030   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1031   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1032           || ip0->protocol == IP_PROTOCOL_ICMP6
1033           || ip0->protocol == IP_PROTOCOL_UDP
1034           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1035
1036   udp0 = (void *) (ip0 + 1);
1037   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1038     {
1039       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1040                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1041       return p0->flags;
1042     }
1043
1044   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1045
1046   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1047                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1048
1049   return p0->flags;
1050 }
1051 #endif
1052
1053 /**
1054  * @brief returns number of links on which src is reachable.
1055  */
1056 always_inline int
1057 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1058 {
1059   const load_balance_t *lb0;
1060   index_t lbi;
1061   u32 fib_index;
1062
1063   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1064                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1065   fib_index =
1066     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1067     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1068
1069   lbi = ip6_fib_table_fwding_lookup (im, fib_index, &i->src_address);
1070   lb0 = load_balance_get (lbi);
1071
1072   return (fib_urpf_check_size (lb0->lb_urpf));
1073 }
1074
1075 always_inline u8
1076 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1077                            u32 * udp_offset0)
1078 {
1079   u32 proto0;
1080   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1081   if (proto0 != IP_PROTOCOL_UDP)
1082     {
1083       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1084       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1085     }
1086   return proto0;
1087 }
1088
1089 /* *INDENT-OFF* */
1090 VNET_FEATURE_ARC_INIT (ip6_local) =
1091 {
1092   .arc_name  = "ip6-local",
1093   .start_nodes = VNET_FEATURES ("ip6-local"),
1094 };
1095 /* *INDENT-ON* */
1096
1097 always_inline uword
1098 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1099                   vlib_frame_t * frame, int head_of_feature_arc)
1100 {
1101   ip6_main_t *im = &ip6_main;
1102   ip_lookup_main_t *lm = &im->lookup_main;
1103   u32 *from, n_left_from;
1104   vlib_node_runtime_t *error_node =
1105     vlib_node_get_runtime (vm, ip6_input_node.index);
1106   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1107   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1108   u16 nexts[VLIB_FRAME_SIZE], *next;
1109
1110   from = vlib_frame_vector_args (frame);
1111   n_left_from = frame->n_vectors;
1112
1113   if (node->flags & VLIB_NODE_FLAG_TRACE)
1114     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1115
1116   vlib_get_buffers (vm, from, bufs, n_left_from);
1117   b = bufs;
1118   next = nexts;
1119
1120   while (n_left_from > 2)
1121     {
1122       /* Prefetch next iteration. */
1123       if (n_left_from >= 6)
1124         {
1125           vlib_prefetch_buffer_header (b[4], STORE);
1126           vlib_prefetch_buffer_header (b[5], STORE);
1127           vlib_prefetch_buffer_data (b[2], LOAD);
1128           vlib_prefetch_buffer_data (b[3], LOAD);
1129         }
1130
1131       u8 error[2];
1132       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1133       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1134
1135       ip6_header_t *ip[2];
1136       ip[0] = vlib_buffer_get_current (b[0]);
1137       ip[1] = vlib_buffer_get_current (b[1]);
1138
1139       if (head_of_feature_arc)
1140         {
1141           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1142           vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1143
1144           u8 type[2];
1145           type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol];
1146           type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol];
1147
1148           u32 flags[2];
1149           flags[0] = b[0]->flags;
1150           flags[1] = b[1]->flags;
1151
1152           u32 good_l4_csum[2];
1153           good_l4_csum[0] =
1154             flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1155                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1156                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1157           good_l4_csum[1] =
1158             flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1159                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1160                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1161
1162           u32 udp_offset[2] = { };
1163           u8 is_tcp_udp[2];
1164           is_tcp_udp[0] =
1165             ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]);
1166           is_tcp_udp[1] =
1167             ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]);
1168           i16 len_diff[2] = { 0 };
1169           if (PREDICT_TRUE (is_tcp_udp[0]))
1170             {
1171               udp_header_t *udp =
1172                 (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]);
1173               good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP
1174                 && udp->checksum == 0;
1175               /* optimistically verify UDP length. */
1176               u16 ip_len, udp_len;
1177               ip_len = clib_net_to_host_u16 (ip[0]->payload_length);
1178               udp_len = clib_net_to_host_u16 (udp->length);
1179               len_diff[0] = ip_len - udp_len;
1180             }
1181           if (PREDICT_TRUE (is_tcp_udp[1]))
1182             {
1183               udp_header_t *udp =
1184                 (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]);
1185               good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP
1186                 && udp->checksum == 0;
1187               /* optimistically verify UDP length. */
1188               u16 ip_len, udp_len;
1189               ip_len = clib_net_to_host_u16 (ip[1]->payload_length);
1190               udp_len = clib_net_to_host_u16 (udp->length);
1191               len_diff[1] = ip_len - udp_len;
1192             }
1193
1194           good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1195           good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1196
1197           len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0;
1198           len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0;
1199
1200           u8 need_csum[2];
1201           need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN
1202             && !good_l4_csum[0]
1203             && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1204           need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN
1205             && !good_l4_csum[1]
1206             && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1207           if (PREDICT_FALSE (need_csum[0]))
1208             {
1209               flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1210               good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1211             }
1212           if (PREDICT_FALSE (need_csum[1]))
1213             {
1214               flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]);
1215               good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1216             }
1217
1218           error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1219           error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0];
1220           error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1221           error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1];
1222
1223           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1224                          IP6_ERROR_UDP_CHECKSUM,
1225                          "Wrong IP6 errors constants");
1226           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1227                          IP6_ERROR_ICMP_CHECKSUM,
1228                          "Wrong IP6 errors constants");
1229
1230           error[0] =
1231             !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0];
1232           error[1] =
1233             !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1];
1234
1235           /* Drop packets from unroutable hosts. */
1236           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1237           u8 unroutable[2];
1238           unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL
1239             && type[0] != IP_BUILTIN_PROTOCOL_ICMP
1240             && !ip6_address_is_link_local_unicast (&ip[0]->src_address);
1241           unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL
1242             && type[1] != IP_BUILTIN_PROTOCOL_ICMP
1243             && !ip6_address_is_link_local_unicast (&ip[1]->src_address);
1244           if (PREDICT_FALSE (unroutable[0]))
1245             {
1246               error[0] =
1247                 !ip6_urpf_loose_check (im, b[0],
1248                                        ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS
1249                 : error[0];
1250             }
1251           if (PREDICT_FALSE (unroutable[1]))
1252             {
1253               error[1] =
1254                 !ip6_urpf_loose_check (im, b[1],
1255                                        ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS
1256                 : error[1];
1257             }
1258
1259           vnet_buffer (b[0])->ip.fib_index =
1260             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1261             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1262             vnet_buffer (b[0])->ip.fib_index;
1263           vnet_buffer (b[1])->ip.fib_index =
1264             vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1265             vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1266             vnet_buffer (b[1])->ip.fib_index;
1267         }                       /* head_of_feature_arc */
1268
1269       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
1270       next[0] =
1271         error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1272       next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol];
1273       next[1] =
1274         error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1];
1275
1276       b[0]->error = error_node->errors[0];
1277       b[1]->error = error_node->errors[1];
1278
1279       if (head_of_feature_arc)
1280         {
1281           u8 ip6_unknown[2];
1282           ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1283           ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1284           if (PREDICT_TRUE (ip6_unknown[0]))
1285             {
1286               u32 next32 = next[0];
1287               vnet_feature_arc_start (arc_index,
1288                                       vnet_buffer (b[0])->sw_if_index
1289                                       [VLIB_RX], &next32, b[0]);
1290               next[0] = next32;
1291             }
1292           if (PREDICT_TRUE (ip6_unknown[1]))
1293             {
1294               u32 next32 = next[1];
1295               vnet_feature_arc_start (arc_index,
1296                                       vnet_buffer (b[1])->sw_if_index
1297                                       [VLIB_RX], &next32, b[1]);
1298               next[1] = next32;
1299             }
1300         }
1301
1302       /* next */
1303       b += 2;
1304       next += 2;
1305       n_left_from -= 2;
1306     }
1307
1308   while (n_left_from)
1309     {
1310       u8 error;
1311       error = IP6_ERROR_UNKNOWN_PROTOCOL;
1312
1313       ip6_header_t *ip;
1314       ip = vlib_buffer_get_current (b[0]);
1315
1316       if (head_of_feature_arc)
1317         {
1318           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1319           u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
1320
1321           u32 flags = b[0]->flags;
1322           u32 good_l4_csum =
1323             flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1324                      VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1325                      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1326
1327           u32 udp_offset;
1328           i16 len_diff = 0;
1329           u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
1330           if (PREDICT_TRUE (is_tcp_udp))
1331             {
1332               udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset);
1333               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1334               good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP
1335                 && udp->checksum == 0;
1336               /* optimistically verify UDP length. */
1337               u16 ip_len, udp_len;
1338               ip_len = clib_net_to_host_u16 (ip->payload_length);
1339               udp_len = clib_net_to_host_u16 (udp->length);
1340               len_diff = ip_len - udp_len;
1341             }
1342
1343           good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
1344           len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
1345
1346           u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
1347             && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1348           if (PREDICT_FALSE (need_csum))
1349             {
1350               flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1351               good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1352             }
1353
1354           error = IP6_ERROR_UNKNOWN_PROTOCOL;
1355           error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
1356
1357           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1358                          IP6_ERROR_UDP_CHECKSUM,
1359                          "Wrong IP6 errors constants");
1360           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1361                          IP6_ERROR_ICMP_CHECKSUM,
1362                          "Wrong IP6 errors constants");
1363
1364           error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error;
1365
1366           /* Drop packets from unroutable hosts. */
1367           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1368           u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL
1369             && type != IP_BUILTIN_PROTOCOL_ICMP
1370             && !ip6_address_is_link_local_unicast (&ip->src_address);
1371           if (PREDICT_FALSE (unroutable))
1372             {
1373               error =
1374                 !ip6_urpf_loose_check (im, b[0],
1375                                        ip) ? IP6_ERROR_SRC_LOOKUP_MISS :
1376                 error;
1377             }
1378
1379           vnet_buffer (b[0])->ip.fib_index =
1380             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1381             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1382             vnet_buffer (b[0])->ip.fib_index;
1383         }                       /* head_of_feature_arc */
1384
1385       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
1386       next[0] =
1387         error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1388
1389       b[0]->error = error_node->errors[0];
1390
1391       if (head_of_feature_arc)
1392         {
1393           if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1394             {
1395               u32 next32 = next[0];
1396               vnet_feature_arc_start (arc_index,
1397                                       vnet_buffer (b[0])->sw_if_index
1398                                       [VLIB_RX], &next32, b[0]);
1399               next[0] = next32;
1400             }
1401         }
1402
1403       /* next */
1404       b += 1;
1405       next += 1;
1406       n_left_from -= 1;
1407     }
1408
1409   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1410   return frame->n_vectors;
1411 }
1412
1413 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1414                                vlib_frame_t * frame)
1415 {
1416   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1417 }
1418
1419 /* *INDENT-OFF* */
1420 VLIB_REGISTER_NODE (ip6_local_node) =
1421 {
1422   .name = "ip6-local",
1423   .vector_size = sizeof (u32),
1424   .format_trace = format_ip6_forward_next_trace,
1425   .n_next_nodes = IP_LOCAL_N_NEXT,
1426   .next_nodes =
1427   {
1428     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1429     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1430     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1431     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1432     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-reassembly",
1433   },
1434 };
1435 /* *INDENT-ON* */
1436
1437 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
1438                                           vlib_node_runtime_t * node,
1439                                           vlib_frame_t * frame)
1440 {
1441   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1442 }
1443
1444 /* *INDENT-OFF* */
1445 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
1446   .name = "ip6-local-end-of-arc",
1447   .vector_size = sizeof (u32),
1448
1449   .format_trace = format_ip6_forward_next_trace,
1450   .sibling_of = "ip6-local",
1451 };
1452
1453 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1454   .arc_name = "ip6-local",
1455   .node_name = "ip6-local-end-of-arc",
1456   .runs_before = 0, /* not before any other features */
1457 };
1458 /* *INDENT-ON* */
1459
1460 #ifdef CLIB_MARCH_VARIANT
1461 extern vlib_node_registration_t ip6_local_node;
1462
1463 #else
1464
1465 void
1466 ip6_register_protocol (u32 protocol, u32 node_index)
1467 {
1468   vlib_main_t *vm = vlib_get_main ();
1469   ip6_main_t *im = &ip6_main;
1470   ip_lookup_main_t *lm = &im->lookup_main;
1471
1472   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1473   lm->local_next_by_ip_protocol[protocol] =
1474     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1475 }
1476
1477 clib_error_t *
1478 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index,
1479                     u8 refresh)
1480 {
1481   vnet_main_t *vnm = vnet_get_main ();
1482   ip6_main_t *im = &ip6_main;
1483   icmp6_neighbor_solicitation_header_t *h;
1484   ip6_address_t *src;
1485   ip_interface_address_t *ia;
1486   ip_adjacency_t *adj;
1487   vnet_hw_interface_t *hi;
1488   vnet_sw_interface_t *si;
1489   vlib_buffer_t *b;
1490   adj_index_t ai;
1491   u32 bi = 0;
1492   int bogus_length;
1493
1494   si = vnet_get_sw_interface (vnm, sw_if_index);
1495
1496   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1497     {
1498       return clib_error_return (0, "%U: interface %U down",
1499                                 format_ip6_address, dst,
1500                                 format_vnet_sw_if_index_name, vnm,
1501                                 sw_if_index);
1502     }
1503
1504   src =
1505     ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1506   if (!src)
1507     {
1508       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1509       return clib_error_return
1510         (0, "no matching interface address for destination %U (interface %U)",
1511          format_ip6_address, dst,
1512          format_vnet_sw_if_index_name, vnm, sw_if_index);
1513     }
1514
1515   h =
1516     vlib_packet_template_get_packet (vm,
1517                                      &im->discover_neighbor_packet_template,
1518                                      &bi);
1519   if (!h)
1520     return clib_error_return (0, "ICMP6 NS packet allocation failed");
1521
1522   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1523
1524   /* Destination address is a solicited node multicast address.  We need to fill in
1525      the low 24 bits with low 24 bits of target's address. */
1526   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1527   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1528   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1529
1530   h->ip.src_address = src[0];
1531   h->neighbor.target_address = dst[0];
1532
1533   if (PREDICT_FALSE (!hi->hw_address))
1534     {
1535       return clib_error_return (0, "%U: interface %U do not support ip probe",
1536                                 format_ip6_address, dst,
1537                                 format_vnet_sw_if_index_name, vnm,
1538                                 sw_if_index);
1539     }
1540
1541   clib_memcpy_fast (h->link_layer_option.ethernet_address, hi->hw_address,
1542                     vec_len (hi->hw_address));
1543
1544   h->neighbor.icmp.checksum =
1545     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1546   ASSERT (bogus_length == 0);
1547
1548   b = vlib_get_buffer (vm, bi);
1549   vnet_buffer (b)->sw_if_index[VLIB_RX] =
1550     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1551
1552   /* Add encapsulation string for software interface (e.g. ethernet header). */
1553   ip46_address_t nh = {
1554     .ip6 = *dst,
1555   };
1556
1557   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP6,
1558                             VNET_LINK_IP6, &nh, sw_if_index);
1559   adj = adj_get (ai);
1560
1561   /* Peer has been previously resolved, retrieve glean adj instead */
1562   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE && refresh == 0)
1563     {
1564       adj_unlock (ai);
1565       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP6,
1566                                   VNET_LINK_IP6, sw_if_index, &nh);
1567       adj = adj_get (ai);
1568     }
1569
1570   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1571   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1572
1573   {
1574     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
1575     u32 *to_next = vlib_frame_vector_args (f);
1576     to_next[0] = bi;
1577     f->n_vectors = 1;
1578     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1579   }
1580
1581   adj_unlock (ai);
1582   return /* no error */ 0;
1583 }
1584 #endif
1585
1586 typedef enum
1587 {
1588   IP6_REWRITE_NEXT_DROP,
1589   IP6_REWRITE_NEXT_ICMP_ERROR,
1590   IP6_REWRITE_NEXT_FRAGMENT,
1591   IP6_REWRITE_N_NEXT            /* Last */
1592 } ip6_rewrite_next_t;
1593
1594 /**
1595  * This bits of an IPv6 address to mask to construct a multicast
1596  * MAC address
1597  */
1598 #define IP6_MCAST_ADDR_MASK 0xffffffff
1599
1600 always_inline void
1601 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1602                u16 adj_packet_bytes, bool is_locally_generated,
1603                u32 * next, u32 * error)
1604 {
1605   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1606     {
1607       if (is_locally_generated)
1608         {
1609           /* IP fragmentation */
1610           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1611                                    IP6_FRAG_NEXT_IP6_REWRITE, 0);
1612           *next = IP6_REWRITE_NEXT_FRAGMENT;
1613           *error = IP6_ERROR_MTU_EXCEEDED;
1614         }
1615       else
1616         {
1617           *error = IP6_ERROR_MTU_EXCEEDED;
1618           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1619                                        adj_packet_bytes);
1620           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1621         }
1622     }
1623 }
1624
1625 always_inline uword
1626 ip6_rewrite_inline_with_gso (vlib_main_t * vm,
1627                              vlib_node_runtime_t * node,
1628                              vlib_frame_t * frame,
1629                              int do_counters, int is_midchain, int is_mcast,
1630                              int do_gso)
1631 {
1632   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1633   u32 *from = vlib_frame_vector_args (frame);
1634   u32 n_left_from, n_left_to_next, *to_next, next_index;
1635   vlib_node_runtime_t *error_node =
1636     vlib_node_get_runtime (vm, ip6_input_node.index);
1637
1638   n_left_from = frame->n_vectors;
1639   next_index = node->cached_next_index;
1640   u32 thread_index = vm->thread_index;
1641
1642   while (n_left_from > 0)
1643     {
1644       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1645
1646       while (n_left_from >= 4 && n_left_to_next >= 2)
1647         {
1648           ip_adjacency_t *adj0, *adj1;
1649           vlib_buffer_t *p0, *p1;
1650           ip6_header_t *ip0, *ip1;
1651           u32 pi0, rw_len0, next0, error0, adj_index0;
1652           u32 pi1, rw_len1, next1, error1, adj_index1;
1653           u32 tx_sw_if_index0, tx_sw_if_index1;
1654           bool is_locally_originated0, is_locally_originated1;
1655
1656           /* Prefetch next iteration. */
1657           {
1658             vlib_buffer_t *p2, *p3;
1659
1660             p2 = vlib_get_buffer (vm, from[2]);
1661             p3 = vlib_get_buffer (vm, from[3]);
1662
1663             vlib_prefetch_buffer_header (p2, LOAD);
1664             vlib_prefetch_buffer_header (p3, LOAD);
1665
1666             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1667             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1668
1669             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1670             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1671           }
1672
1673           pi0 = to_next[0] = from[0];
1674           pi1 = to_next[1] = from[1];
1675
1676           from += 2;
1677           n_left_from -= 2;
1678           to_next += 2;
1679           n_left_to_next -= 2;
1680
1681           p0 = vlib_get_buffer (vm, pi0);
1682           p1 = vlib_get_buffer (vm, pi1);
1683
1684           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1685           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1686
1687           ip0 = vlib_buffer_get_current (p0);
1688           ip1 = vlib_buffer_get_current (p1);
1689
1690           error0 = error1 = IP6_ERROR_NONE;
1691           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1692
1693           is_locally_originated0 =
1694             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1695           if (PREDICT_TRUE (!is_locally_originated0))
1696             {
1697               i32 hop_limit0 = ip0->hop_limit;
1698
1699               /* Input node should have reject packets with hop limit 0. */
1700               ASSERT (ip0->hop_limit > 0);
1701
1702               hop_limit0 -= 1;
1703
1704               ip0->hop_limit = hop_limit0;
1705
1706               /*
1707                * If the hop count drops below 1 when forwarding, generate
1708                * an ICMP response.
1709                */
1710               if (PREDICT_FALSE (hop_limit0 <= 0))
1711                 {
1712                   error0 = IP6_ERROR_TIME_EXPIRED;
1713                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1714                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1715                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1716                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1717                                                0);
1718                 }
1719             }
1720           else
1721             {
1722               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1723             }
1724           is_locally_originated1 =
1725             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1726           if (PREDICT_TRUE (!is_locally_originated1))
1727             {
1728               i32 hop_limit1 = ip1->hop_limit;
1729
1730               /* Input node should have reject packets with hop limit 0. */
1731               ASSERT (ip1->hop_limit > 0);
1732
1733               hop_limit1 -= 1;
1734
1735               ip1->hop_limit = hop_limit1;
1736
1737               /*
1738                * If the hop count drops below 1 when forwarding, generate
1739                * an ICMP response.
1740                */
1741               if (PREDICT_FALSE (hop_limit1 <= 0))
1742                 {
1743                   error1 = IP6_ERROR_TIME_EXPIRED;
1744                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1745                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1746                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1747                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1748                                                0);
1749                 }
1750             }
1751           else
1752             {
1753               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1754             }
1755           adj0 = adj_get (adj_index0);
1756           adj1 = adj_get (adj_index1);
1757
1758           rw_len0 = adj0[0].rewrite_header.data_bytes;
1759           rw_len1 = adj1[0].rewrite_header.data_bytes;
1760           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1761           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1762
1763           if (do_counters)
1764             {
1765               vlib_increment_combined_counter
1766                 (&adjacency_counters,
1767                  thread_index, adj_index0, 1,
1768                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1769               vlib_increment_combined_counter
1770                 (&adjacency_counters,
1771                  thread_index, adj_index1, 1,
1772                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1773             }
1774
1775           /* Check MTU of outgoing interface. */
1776           u16 ip0_len =
1777             clib_net_to_host_u16 (ip0->payload_length) +
1778             sizeof (ip6_header_t);
1779           u16 ip1_len =
1780             clib_net_to_host_u16 (ip1->payload_length) +
1781             sizeof (ip6_header_t);
1782           if (do_gso && (p0->flags & VNET_BUFFER_F_GSO))
1783             ip0_len = gso_mtu_sz (p0);
1784           if (do_gso && (p1->flags & VNET_BUFFER_F_GSO))
1785             ip1_len = gso_mtu_sz (p1);
1786
1787
1788
1789           ip6_mtu_check (p0, ip0_len,
1790                          adj0[0].rewrite_header.max_l3_packet_bytes,
1791                          is_locally_originated0, &next0, &error0);
1792           ip6_mtu_check (p1, ip1_len,
1793                          adj1[0].rewrite_header.max_l3_packet_bytes,
1794                          is_locally_originated1, &next1, &error1);
1795
1796           /* Don't adjust the buffer for hop count issue; icmp-error node
1797            * wants to see the IP header */
1798           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1799             {
1800               p0->current_data -= rw_len0;
1801               p0->current_length += rw_len0;
1802
1803               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1804               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1805               next0 = adj0[0].rewrite_header.next_index;
1806
1807               if (PREDICT_FALSE
1808                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1809                 vnet_feature_arc_start (lm->output_feature_arc_index,
1810                                         tx_sw_if_index0, &next0, p0);
1811             }
1812           else
1813             {
1814               p0->error = error_node->errors[error0];
1815             }
1816           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1817             {
1818               p1->current_data -= rw_len1;
1819               p1->current_length += rw_len1;
1820
1821               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1822               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1823               next1 = adj1[0].rewrite_header.next_index;
1824
1825               if (PREDICT_FALSE
1826                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1827                 vnet_feature_arc_start (lm->output_feature_arc_index,
1828                                         tx_sw_if_index1, &next1, p1);
1829             }
1830           else
1831             {
1832               p1->error = error_node->errors[error1];
1833             }
1834
1835           if (is_midchain)
1836             {
1837               /* before we paint on the next header, update the L4
1838                * checksums if required, since there's no offload on a tunnel */
1839               calc_checksums (vm, p0);
1840               calc_checksums (vm, p1);
1841             }
1842
1843           /* Guess we are only writing on simple Ethernet header. */
1844           vnet_rewrite_two_headers (adj0[0], adj1[0],
1845                                     ip0, ip1, sizeof (ethernet_header_t));
1846
1847           if (is_midchain)
1848             {
1849               if (adj0->sub_type.midchain.fixup_func)
1850                 adj0->sub_type.midchain.fixup_func
1851                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1852               if (adj1->sub_type.midchain.fixup_func)
1853                 adj1->sub_type.midchain.fixup_func
1854                   (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1855             }
1856           if (is_mcast)
1857             {
1858               /*
1859                * copy bytes from the IP address into the MAC rewrite
1860                */
1861               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1862                                           adj0->
1863                                           rewrite_header.dst_mcast_offset,
1864                                           &ip0->dst_address.as_u32[3],
1865                                           (u8 *) ip0);
1866               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1867                                           adj1->
1868                                           rewrite_header.dst_mcast_offset,
1869                                           &ip1->dst_address.as_u32[3],
1870                                           (u8 *) ip1);
1871             }
1872
1873           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1874                                            to_next, n_left_to_next,
1875                                            pi0, pi1, next0, next1);
1876         }
1877
1878       while (n_left_from > 0 && n_left_to_next > 0)
1879         {
1880           ip_adjacency_t *adj0;
1881           vlib_buffer_t *p0;
1882           ip6_header_t *ip0;
1883           u32 pi0, rw_len0;
1884           u32 adj_index0, next0, error0;
1885           u32 tx_sw_if_index0;
1886           bool is_locally_originated0;
1887
1888           pi0 = to_next[0] = from[0];
1889
1890           p0 = vlib_get_buffer (vm, pi0);
1891
1892           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1893
1894           adj0 = adj_get (adj_index0);
1895
1896           ip0 = vlib_buffer_get_current (p0);
1897
1898           error0 = IP6_ERROR_NONE;
1899           next0 = IP6_REWRITE_NEXT_DROP;
1900
1901           /* Check hop limit */
1902           is_locally_originated0 =
1903             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1904           if (PREDICT_TRUE (!is_locally_originated0))
1905             {
1906               i32 hop_limit0 = ip0->hop_limit;
1907
1908               ASSERT (ip0->hop_limit > 0);
1909
1910               hop_limit0 -= 1;
1911
1912               ip0->hop_limit = hop_limit0;
1913
1914               if (PREDICT_FALSE (hop_limit0 <= 0))
1915                 {
1916                   /*
1917                    * If the hop count drops below 1 when forwarding, generate
1918                    * an ICMP response.
1919                    */
1920                   error0 = IP6_ERROR_TIME_EXPIRED;
1921                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1922                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1923                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1924                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1925                                                0);
1926                 }
1927             }
1928           else
1929             {
1930               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1931             }
1932
1933           if (is_midchain)
1934             {
1935               calc_checksums (vm, p0);
1936             }
1937
1938           /* Guess we are only writing on simple Ethernet header. */
1939           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
1940
1941           /* Update packet buffer attributes/set output interface. */
1942           rw_len0 = adj0[0].rewrite_header.data_bytes;
1943           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1944
1945           if (do_counters)
1946             {
1947               vlib_increment_combined_counter
1948                 (&adjacency_counters,
1949                  thread_index, adj_index0, 1,
1950                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1951             }
1952
1953           /* Check MTU of outgoing interface. */
1954           u16 ip0_len =
1955             clib_net_to_host_u16 (ip0->payload_length) +
1956             sizeof (ip6_header_t);
1957           if (do_gso && (p0->flags & VNET_BUFFER_F_GSO))
1958             ip0_len = gso_mtu_sz (p0);
1959
1960           ip6_mtu_check (p0, ip0_len,
1961                          adj0[0].rewrite_header.max_l3_packet_bytes,
1962                          is_locally_originated0, &next0, &error0);
1963
1964           /* Don't adjust the buffer for hop count issue; icmp-error node
1965            * wants to see the IP header */
1966           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1967             {
1968               p0->current_data -= rw_len0;
1969               p0->current_length += rw_len0;
1970
1971               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1972
1973               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1974               next0 = adj0[0].rewrite_header.next_index;
1975
1976               if (PREDICT_FALSE
1977                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1978                 vnet_feature_arc_start (lm->output_feature_arc_index,
1979                                         tx_sw_if_index0, &next0, p0);
1980             }
1981           else
1982             {
1983               p0->error = error_node->errors[error0];
1984             }
1985
1986           if (is_midchain)
1987             {
1988               if (adj0->sub_type.midchain.fixup_func)
1989                 adj0->sub_type.midchain.fixup_func
1990                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1991             }
1992           if (is_mcast)
1993             {
1994               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1995                                           adj0->
1996                                           rewrite_header.dst_mcast_offset,
1997                                           &ip0->dst_address.as_u32[3],
1998                                           (u8 *) ip0);
1999             }
2000
2001           from += 1;
2002           n_left_from -= 1;
2003           to_next += 1;
2004           n_left_to_next -= 1;
2005
2006           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2007                                            to_next, n_left_to_next,
2008                                            pi0, next0);
2009         }
2010
2011       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2012     }
2013
2014   /* Need to do trace after rewrites to pick up new packet data. */
2015   if (node->flags & VLIB_NODE_FLAG_TRACE)
2016     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2017
2018   return frame->n_vectors;
2019 }
2020
2021 always_inline uword
2022 ip6_rewrite_inline (vlib_main_t * vm,
2023                     vlib_node_runtime_t * node,
2024                     vlib_frame_t * frame,
2025                     int do_counters, int is_midchain, int is_mcast)
2026 {
2027   vnet_main_t *vnm = vnet_get_main ();
2028   if (PREDICT_FALSE (vnm->interface_main.gso_interface_count > 0))
2029     return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2030                                         is_midchain, is_mcast,
2031                                         1 /* do_gso */ );
2032   else
2033     return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2034                                         is_midchain, is_mcast,
2035                                         0 /* no do_gso */ );
2036 }
2037
2038 VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm,
2039                                  vlib_node_runtime_t * node,
2040                                  vlib_frame_t * frame)
2041 {
2042   if (adj_are_counters_enabled ())
2043     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2044   else
2045     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2046 }
2047
2048 VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm,
2049                                        vlib_node_runtime_t * node,
2050                                        vlib_frame_t * frame)
2051 {
2052   if (adj_are_counters_enabled ())
2053     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2054   else
2055     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2056 }
2057
2058 VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm,
2059                                        vlib_node_runtime_t * node,
2060                                        vlib_frame_t * frame)
2061 {
2062   if (adj_are_counters_enabled ())
2063     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2064   else
2065     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2066 }
2067
2068 VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm,
2069                                   vlib_node_runtime_t * node,
2070                                   vlib_frame_t * frame)
2071 {
2072   if (adj_are_counters_enabled ())
2073     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2074   else
2075     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2076 }
2077
2078 VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
2079                                         vlib_node_runtime_t * node,
2080                                         vlib_frame_t * frame)
2081 {
2082   if (adj_are_counters_enabled ())
2083     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2084   else
2085     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2086 }
2087
2088 /* *INDENT-OFF* */
2089 VLIB_REGISTER_NODE (ip6_midchain_node) =
2090 {
2091   .name = "ip6-midchain",
2092   .vector_size = sizeof (u32),
2093   .format_trace = format_ip6_forward_next_trace,
2094   .sibling_of = "ip6-rewrite",
2095   };
2096
2097 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2098 {
2099   .name = "ip6-rewrite",
2100   .vector_size = sizeof (u32),
2101   .format_trace = format_ip6_rewrite_trace,
2102   .n_next_nodes = IP6_REWRITE_N_NEXT,
2103   .next_nodes =
2104   {
2105     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2106     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2107     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2108   },
2109 };
2110
2111 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2112   .name = "ip6-rewrite-bcast",
2113   .vector_size = sizeof (u32),
2114
2115   .format_trace = format_ip6_rewrite_trace,
2116   .sibling_of = "ip6-rewrite",
2117 };
2118
2119 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2120 {
2121   .name = "ip6-rewrite-mcast",
2122   .vector_size = sizeof (u32),
2123   .format_trace = format_ip6_rewrite_trace,
2124   .sibling_of = "ip6-rewrite",
2125 };
2126
2127
2128 VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
2129 {
2130   .name = "ip6-mcast-midchain",
2131   .vector_size = sizeof (u32),
2132   .format_trace = format_ip6_rewrite_trace,
2133   .sibling_of = "ip6-rewrite",
2134 };
2135
2136 /* *INDENT-ON* */
2137
2138 /*
2139  * Hop-by-Hop handling
2140  */
2141 #ifndef CLIB_MARCH_VARIANT
2142 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2143 #endif /* CLIB_MARCH_VARIANT */
2144
2145 #define foreach_ip6_hop_by_hop_error \
2146 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2147 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2148 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2149
2150 /* *INDENT-OFF* */
2151 typedef enum
2152 {
2153 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2154   foreach_ip6_hop_by_hop_error
2155 #undef _
2156   IP6_HOP_BY_HOP_N_ERROR,
2157 } ip6_hop_by_hop_error_t;
2158 /* *INDENT-ON* */
2159
2160 /*
2161  * Primary h-b-h handler trace support
2162  * We work pretty hard on the problem for obvious reasons
2163  */
2164 typedef struct
2165 {
2166   u32 next_index;
2167   u32 trace_len;
2168   u8 option_data[256];
2169 } ip6_hop_by_hop_trace_t;
2170
2171 extern vlib_node_registration_t ip6_hop_by_hop_node;
2172
2173 static char *ip6_hop_by_hop_error_strings[] = {
2174 #define _(sym,string) string,
2175   foreach_ip6_hop_by_hop_error
2176 #undef _
2177 };
2178
2179 #ifndef CLIB_MARCH_VARIANT
2180 u8 *
2181 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2182 {
2183   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2184   int total_len = va_arg (*args, int);
2185   ip6_hop_by_hop_option_t *opt0, *limit0;
2186   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2187   u8 type0;
2188
2189   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2190               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2191
2192   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2193   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2194
2195   while (opt0 < limit0)
2196     {
2197       type0 = opt0->type;
2198       switch (type0)
2199         {
2200         case 0:         /* Pad, just stop */
2201           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2202           break;
2203
2204         default:
2205           if (hm->trace[type0])
2206             {
2207               s = (*hm->trace[type0]) (s, opt0);
2208             }
2209           else
2210             {
2211               s =
2212                 format (s, "\n    unrecognized option %d length %d", type0,
2213                         opt0->length);
2214             }
2215           opt0 =
2216             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2217                                          sizeof (ip6_hop_by_hop_option_t));
2218           break;
2219         }
2220     }
2221   return s;
2222 }
2223 #endif
2224
2225 static u8 *
2226 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2227 {
2228   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2229   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2230   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2231   ip6_hop_by_hop_header_t *hbh0;
2232   ip6_hop_by_hop_option_t *opt0, *limit0;
2233   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2234
2235   u8 type0;
2236
2237   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2238
2239   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2240               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2241
2242   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2243   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2244
2245   while (opt0 < limit0)
2246     {
2247       type0 = opt0->type;
2248       switch (type0)
2249         {
2250         case 0:         /* Pad, just stop */
2251           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2252           break;
2253
2254         default:
2255           if (hm->trace[type0])
2256             {
2257               s = (*hm->trace[type0]) (s, opt0);
2258             }
2259           else
2260             {
2261               s =
2262                 format (s, "\n    unrecognized option %d length %d", type0,
2263                         opt0->length);
2264             }
2265           opt0 =
2266             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2267                                          sizeof (ip6_hop_by_hop_option_t));
2268           break;
2269         }
2270     }
2271   return s;
2272 }
2273
2274 always_inline u8
2275 ip6_scan_hbh_options (vlib_buffer_t * b0,
2276                       ip6_header_t * ip0,
2277                       ip6_hop_by_hop_header_t * hbh0,
2278                       ip6_hop_by_hop_option_t * opt0,
2279                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2280 {
2281   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2282   u8 type0;
2283   u8 error0 = 0;
2284
2285   while (opt0 < limit0)
2286     {
2287       type0 = opt0->type;
2288       switch (type0)
2289         {
2290         case 0:         /* Pad1 */
2291           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2292           continue;
2293         case 1:         /* PadN */
2294           break;
2295         default:
2296           if (hm->options[type0])
2297             {
2298               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2299                 {
2300                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2301                   return (error0);
2302                 }
2303             }
2304           else
2305             {
2306               /* Unrecognized mandatory option, check the two high order bits */
2307               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2308                 {
2309                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2310                   break;
2311                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2312                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2313                   *next0 = IP_LOOKUP_NEXT_DROP;
2314                   break;
2315                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2316                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2317                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2318                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2319                                                ICMP6_parameter_problem_unrecognized_option,
2320                                                (u8 *) opt0 - (u8 *) ip0);
2321                   break;
2322                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2323                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2324                   if (!ip6_address_is_multicast (&ip0->dst_address))
2325                     {
2326                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2327                       icmp6_error_set_vnet_buffer (b0,
2328                                                    ICMP6_parameter_problem,
2329                                                    ICMP6_parameter_problem_unrecognized_option,
2330                                                    (u8 *) opt0 - (u8 *) ip0);
2331                     }
2332                   else
2333                     {
2334                       *next0 = IP_LOOKUP_NEXT_DROP;
2335                     }
2336                   break;
2337                 }
2338               return (error0);
2339             }
2340         }
2341       opt0 =
2342         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2343                                      sizeof (ip6_hop_by_hop_option_t));
2344     }
2345   return (error0);
2346 }
2347
2348 /*
2349  * Process the Hop-by-Hop Options header
2350  */
2351 VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
2352                                     vlib_node_runtime_t * node,
2353                                     vlib_frame_t * frame)
2354 {
2355   vlib_node_runtime_t *error_node =
2356     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2357   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2358   u32 n_left_from, *from, *to_next;
2359   ip_lookup_next_t next_index;
2360
2361   from = vlib_frame_vector_args (frame);
2362   n_left_from = frame->n_vectors;
2363   next_index = node->cached_next_index;
2364
2365   while (n_left_from > 0)
2366     {
2367       u32 n_left_to_next;
2368
2369       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2370
2371       while (n_left_from >= 4 && n_left_to_next >= 2)
2372         {
2373           u32 bi0, bi1;
2374           vlib_buffer_t *b0, *b1;
2375           u32 next0, next1;
2376           ip6_header_t *ip0, *ip1;
2377           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2378           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2379           u8 error0 = 0, error1 = 0;
2380
2381           /* Prefetch next iteration. */
2382           {
2383             vlib_buffer_t *p2, *p3;
2384
2385             p2 = vlib_get_buffer (vm, from[2]);
2386             p3 = vlib_get_buffer (vm, from[3]);
2387
2388             vlib_prefetch_buffer_header (p2, LOAD);
2389             vlib_prefetch_buffer_header (p3, LOAD);
2390
2391             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2392             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2393           }
2394
2395           /* Speculatively enqueue b0, b1 to the current next frame */
2396           to_next[0] = bi0 = from[0];
2397           to_next[1] = bi1 = from[1];
2398           from += 2;
2399           to_next += 2;
2400           n_left_from -= 2;
2401           n_left_to_next -= 2;
2402
2403           b0 = vlib_get_buffer (vm, bi0);
2404           b1 = vlib_get_buffer (vm, bi1);
2405
2406           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2407           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2408           ip_adjacency_t *adj0 = adj_get (adj_index0);
2409           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2410           ip_adjacency_t *adj1 = adj_get (adj_index1);
2411
2412           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2413           next0 = adj0->lookup_next_index;
2414           next1 = adj1->lookup_next_index;
2415
2416           ip0 = vlib_buffer_get_current (b0);
2417           ip1 = vlib_buffer_get_current (b1);
2418           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2419           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2420           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2421           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2422           limit0 =
2423             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2424                                          ((hbh0->length + 1) << 3));
2425           limit1 =
2426             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2427                                          ((hbh1->length + 1) << 3));
2428
2429           /*
2430            * Basic validity checks
2431            */
2432           if ((hbh0->length + 1) << 3 >
2433               clib_net_to_host_u16 (ip0->payload_length))
2434             {
2435               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2436               next0 = IP_LOOKUP_NEXT_DROP;
2437               goto outdual;
2438             }
2439           /* Scan the set of h-b-h options, process ones that we understand */
2440           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2441
2442           if ((hbh1->length + 1) << 3 >
2443               clib_net_to_host_u16 (ip1->payload_length))
2444             {
2445               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2446               next1 = IP_LOOKUP_NEXT_DROP;
2447               goto outdual;
2448             }
2449           /* Scan the set of h-b-h options, process ones that we understand */
2450           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2451
2452         outdual:
2453           /* Has the classifier flagged this buffer for special treatment? */
2454           if (PREDICT_FALSE
2455               ((error0 == 0)
2456                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2457             next0 = hm->next_override;
2458
2459           /* Has the classifier flagged this buffer for special treatment? */
2460           if (PREDICT_FALSE
2461               ((error1 == 0)
2462                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2463             next1 = hm->next_override;
2464
2465           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2466             {
2467               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2468                 {
2469                   ip6_hop_by_hop_trace_t *t =
2470                     vlib_add_trace (vm, node, b0, sizeof (*t));
2471                   u32 trace_len = (hbh0->length + 1) << 3;
2472                   t->next_index = next0;
2473                   /* Capture the h-b-h option verbatim */
2474                   trace_len =
2475                     trace_len <
2476                     ARRAY_LEN (t->option_data) ? trace_len :
2477                     ARRAY_LEN (t->option_data);
2478                   t->trace_len = trace_len;
2479                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2480                 }
2481               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2482                 {
2483                   ip6_hop_by_hop_trace_t *t =
2484                     vlib_add_trace (vm, node, b1, sizeof (*t));
2485                   u32 trace_len = (hbh1->length + 1) << 3;
2486                   t->next_index = next1;
2487                   /* Capture the h-b-h option verbatim */
2488                   trace_len =
2489                     trace_len <
2490                     ARRAY_LEN (t->option_data) ? trace_len :
2491                     ARRAY_LEN (t->option_data);
2492                   t->trace_len = trace_len;
2493                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2494                 }
2495
2496             }
2497
2498           b0->error = error_node->errors[error0];
2499           b1->error = error_node->errors[error1];
2500
2501           /* verify speculative enqueue, maybe switch current next frame */
2502           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2503                                            n_left_to_next, bi0, bi1, next0,
2504                                            next1);
2505         }
2506
2507       while (n_left_from > 0 && n_left_to_next > 0)
2508         {
2509           u32 bi0;
2510           vlib_buffer_t *b0;
2511           u32 next0;
2512           ip6_header_t *ip0;
2513           ip6_hop_by_hop_header_t *hbh0;
2514           ip6_hop_by_hop_option_t *opt0, *limit0;
2515           u8 error0 = 0;
2516
2517           /* Speculatively enqueue b0 to the current next frame */
2518           bi0 = from[0];
2519           to_next[0] = bi0;
2520           from += 1;
2521           to_next += 1;
2522           n_left_from -= 1;
2523           n_left_to_next -= 1;
2524
2525           b0 = vlib_get_buffer (vm, bi0);
2526           /*
2527            * Default use the next_index from the adjacency.
2528            * A HBH option rarely redirects to a different node
2529            */
2530           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2531           ip_adjacency_t *adj0 = adj_get (adj_index0);
2532           next0 = adj0->lookup_next_index;
2533
2534           ip0 = vlib_buffer_get_current (b0);
2535           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2536           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2537           limit0 =
2538             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2539                                          ((hbh0->length + 1) << 3));
2540
2541           /*
2542            * Basic validity checks
2543            */
2544           if ((hbh0->length + 1) << 3 >
2545               clib_net_to_host_u16 (ip0->payload_length))
2546             {
2547               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2548               next0 = IP_LOOKUP_NEXT_DROP;
2549               goto out0;
2550             }
2551
2552           /* Scan the set of h-b-h options, process ones that we understand */
2553           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2554
2555         out0:
2556           /* Has the classifier flagged this buffer for special treatment? */
2557           if (PREDICT_FALSE
2558               ((error0 == 0)
2559                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2560             next0 = hm->next_override;
2561
2562           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2563             {
2564               ip6_hop_by_hop_trace_t *t =
2565                 vlib_add_trace (vm, node, b0, sizeof (*t));
2566               u32 trace_len = (hbh0->length + 1) << 3;
2567               t->next_index = next0;
2568               /* Capture the h-b-h option verbatim */
2569               trace_len =
2570                 trace_len <
2571                 ARRAY_LEN (t->option_data) ? trace_len :
2572                 ARRAY_LEN (t->option_data);
2573               t->trace_len = trace_len;
2574               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2575             }
2576
2577           b0->error = error_node->errors[error0];
2578
2579           /* verify speculative enqueue, maybe switch current next frame */
2580           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2581                                            n_left_to_next, bi0, next0);
2582         }
2583       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2584     }
2585   return frame->n_vectors;
2586 }
2587
2588 /* *INDENT-OFF* */
2589 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2590 {
2591   .name = "ip6-hop-by-hop",
2592   .sibling_of = "ip6-lookup",
2593   .vector_size = sizeof (u32),
2594   .format_trace = format_ip6_hop_by_hop_trace,
2595   .type = VLIB_NODE_TYPE_INTERNAL,
2596   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2597   .error_strings = ip6_hop_by_hop_error_strings,
2598   .n_next_nodes = 0,
2599 };
2600 /* *INDENT-ON* */
2601
2602 static clib_error_t *
2603 ip6_hop_by_hop_init (vlib_main_t * vm)
2604 {
2605   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2606   clib_memset (hm->options, 0, sizeof (hm->options));
2607   clib_memset (hm->trace, 0, sizeof (hm->trace));
2608   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2609   return (0);
2610 }
2611
2612 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2613
2614 #ifndef CLIB_MARCH_VARIANT
2615 void
2616 ip6_hbh_set_next_override (uword next)
2617 {
2618   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2619
2620   hm->next_override = next;
2621 }
2622
2623 int
2624 ip6_hbh_register_option (u8 option,
2625                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2626                                       ip6_hop_by_hop_option_t * opt),
2627                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2628 {
2629   ip6_main_t *im = &ip6_main;
2630   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2631
2632   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2633
2634   /* Already registered */
2635   if (hm->options[option])
2636     return (-1);
2637
2638   hm->options[option] = options;
2639   hm->trace[option] = trace;
2640
2641   /* Set global variable */
2642   im->hbh_enabled = 1;
2643
2644   return (0);
2645 }
2646
2647 int
2648 ip6_hbh_unregister_option (u8 option)
2649 {
2650   ip6_main_t *im = &ip6_main;
2651   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2652
2653   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2654
2655   /* Not registered */
2656   if (!hm->options[option])
2657     return (-1);
2658
2659   hm->options[option] = NULL;
2660   hm->trace[option] = NULL;
2661
2662   /* Disable global knob if this was the last option configured */
2663   int i;
2664   bool found = false;
2665   for (i = 0; i < 256; i++)
2666     {
2667       if (hm->options[option])
2668         {
2669           found = true;
2670           break;
2671         }
2672     }
2673   if (!found)
2674     im->hbh_enabled = 0;
2675
2676   return (0);
2677 }
2678
2679 /* Global IP6 main. */
2680 ip6_main_t ip6_main;
2681 #endif
2682
2683 static clib_error_t *
2684 ip6_lookup_init (vlib_main_t * vm)
2685 {
2686   ip6_main_t *im = &ip6_main;
2687   clib_error_t *error;
2688   uword i;
2689
2690   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2691     return error;
2692
2693   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2694     {
2695       u32 j, i0, i1;
2696
2697       i0 = i / 32;
2698       i1 = i % 32;
2699
2700       for (j = 0; j < i0; j++)
2701         im->fib_masks[i].as_u32[j] = ~0;
2702
2703       if (i1)
2704         im->fib_masks[i].as_u32[i0] =
2705           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2706     }
2707
2708   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2709
2710   if (im->lookup_table_nbuckets == 0)
2711     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2712
2713   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2714
2715   if (im->lookup_table_size == 0)
2716     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2717
2718   clib_bihash_init_24_8 (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2719                          "ip6 FIB fwding table",
2720                          im->lookup_table_nbuckets, im->lookup_table_size);
2721   clib_bihash_init_24_8 (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2722                          "ip6 FIB non-fwding table",
2723                          im->lookup_table_nbuckets, im->lookup_table_size);
2724   clib_bihash_init_40_8 (&im->ip6_mtable.ip6_mhash,
2725                          "ip6 mFIB table",
2726                          im->lookup_table_nbuckets, im->lookup_table_size);
2727
2728   /* Create FIB with index 0 and table id of 0. */
2729   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2730                                      FIB_SOURCE_DEFAULT_ROUTE);
2731   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2732                                       MFIB_SOURCE_DEFAULT_ROUTE);
2733
2734   {
2735     pg_node_t *pn;
2736     pn = pg_get_node (ip6_lookup_node.index);
2737     pn->unformat_edit = unformat_pg_ip6_header;
2738   }
2739
2740   /* Unless explicitly configured, don't process HBH options */
2741   im->hbh_enabled = 0;
2742
2743   {
2744     icmp6_neighbor_solicitation_header_t p;
2745
2746     clib_memset (&p, 0, sizeof (p));
2747
2748     p.ip.ip_version_traffic_class_and_flow_label =
2749       clib_host_to_net_u32 (0x6 << 28);
2750     p.ip.payload_length =
2751       clib_host_to_net_u16 (sizeof (p) -
2752                             STRUCT_OFFSET_OF
2753                             (icmp6_neighbor_solicitation_header_t, neighbor));
2754     p.ip.protocol = IP_PROTOCOL_ICMP6;
2755     p.ip.hop_limit = 255;
2756     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2757
2758     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2759
2760     p.link_layer_option.header.type =
2761       ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2762     p.link_layer_option.header.n_data_u64s =
2763       sizeof (p.link_layer_option) / sizeof (u64);
2764
2765     vlib_packet_template_init (vm,
2766                                &im->discover_neighbor_packet_template,
2767                                &p, sizeof (p),
2768                                /* alloc chunk size */ 8,
2769                                "ip6 neighbor discovery");
2770   }
2771
2772   return error;
2773 }
2774
2775 VLIB_INIT_FUNCTION (ip6_lookup_init);
2776
2777 static clib_error_t *
2778 test_ip6_link_command_fn (vlib_main_t * vm,
2779                           unformat_input_t * input, vlib_cli_command_t * cmd)
2780 {
2781   u8 mac[6];
2782   ip6_address_t _a, *a = &_a;
2783
2784   if (unformat (input, "%U", unformat_ethernet_address, mac))
2785     {
2786       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2787       vlib_cli_output (vm, "Link local address: %U", format_ip6_address, a);
2788       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2789       vlib_cli_output (vm, "Original MAC address: %U",
2790                        format_ethernet_address, mac);
2791     }
2792
2793   return 0;
2794 }
2795
2796 /*?
2797  * This command converts the given MAC Address into an IPv6 link-local
2798  * address.
2799  *
2800  * @cliexpar
2801  * Example of how to create an IPv6 link-local address:
2802  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2803  * Link local address: fe80::14d9:e0ff:fe91:7986
2804  * Original MAC address: 16:d9:e0:91:79:86
2805  * @cliexend
2806 ?*/
2807 /* *INDENT-OFF* */
2808 VLIB_CLI_COMMAND (test_link_command, static) =
2809 {
2810   .path = "test ip6 link",
2811   .function = test_ip6_link_command_fn,
2812   .short_help = "test ip6 link <mac-address>",
2813 };
2814 /* *INDENT-ON* */
2815
2816 #ifndef CLIB_MARCH_VARIANT
2817 int
2818 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2819 {
2820   u32 fib_index;
2821
2822   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2823
2824   if (~0 == fib_index)
2825     return VNET_API_ERROR_NO_SUCH_FIB;
2826
2827   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2828                                   flow_hash_config);
2829
2830   return 0;
2831 }
2832 #endif
2833
2834 static clib_error_t *
2835 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2836                               unformat_input_t * input,
2837                               vlib_cli_command_t * cmd)
2838 {
2839   int matched = 0;
2840   u32 table_id = 0;
2841   u32 flow_hash_config = 0;
2842   int rv;
2843
2844   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2845     {
2846       if (unformat (input, "table %d", &table_id))
2847         matched = 1;
2848 #define _(a,v) \
2849     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2850       foreach_flow_hash_bit
2851 #undef _
2852         else
2853         break;
2854     }
2855
2856   if (matched == 0)
2857     return clib_error_return (0, "unknown input `%U'",
2858                               format_unformat_error, input);
2859
2860   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2861   switch (rv)
2862     {
2863     case 0:
2864       break;
2865
2866     case -1:
2867       return clib_error_return (0, "no such FIB table %d", table_id);
2868
2869     default:
2870       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2871       break;
2872     }
2873
2874   return 0;
2875 }
2876
2877 /*?
2878  * Configure the set of IPv6 fields used by the flow hash.
2879  *
2880  * @cliexpar
2881  * @parblock
2882  * Example of how to set the flow hash on a given table:
2883  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2884  *
2885  * Example of display the configured flow hash:
2886  * @cliexstart{show ip6 fib}
2887  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2888  * @::/0
2889  *   unicast-ip6-chain
2890  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2891  *     [0] [@0]: dpo-drop ip6
2892  * fe80::/10
2893  *   unicast-ip6-chain
2894  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2895  *     [0] [@2]: dpo-receive
2896  * ff02::1/128
2897  *   unicast-ip6-chain
2898  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2899  *     [0] [@2]: dpo-receive
2900  * ff02::2/128
2901  *   unicast-ip6-chain
2902  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2903  *     [0] [@2]: dpo-receive
2904  * ff02::16/128
2905  *   unicast-ip6-chain
2906  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2907  *     [0] [@2]: dpo-receive
2908  * ff02::1:ff00:0/104
2909  *   unicast-ip6-chain
2910  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2911  *     [0] [@2]: dpo-receive
2912  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2913  * @::/0
2914  *   unicast-ip6-chain
2915  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2916  *     [0] [@0]: dpo-drop ip6
2917  * @::a:1:1:0:4/126
2918  *   unicast-ip6-chain
2919  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2920  *     [0] [@4]: ipv6-glean: af_packet0
2921  * @::a:1:1:0:7/128
2922  *   unicast-ip6-chain
2923  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2924  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2925  * fe80::/10
2926  *   unicast-ip6-chain
2927  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2928  *     [0] [@2]: dpo-receive
2929  * fe80::fe:3eff:fe3e:9222/128
2930  *   unicast-ip6-chain
2931  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2932  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2933  * ff02::1/128
2934  *   unicast-ip6-chain
2935  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2936  *     [0] [@2]: dpo-receive
2937  * ff02::2/128
2938  *   unicast-ip6-chain
2939  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2940  *     [0] [@2]: dpo-receive
2941  * ff02::16/128
2942  *   unicast-ip6-chain
2943  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2944  *     [0] [@2]: dpo-receive
2945  * ff02::1:ff00:0/104
2946  *   unicast-ip6-chain
2947  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2948  *     [0] [@2]: dpo-receive
2949  * @cliexend
2950  * @endparblock
2951 ?*/
2952 /* *INDENT-OFF* */
2953 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
2954 {
2955   .path = "set ip6 flow-hash",
2956   .short_help =
2957   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2958   .function = set_ip6_flow_hash_command_fn,
2959 };
2960 /* *INDENT-ON* */
2961
2962 static clib_error_t *
2963 show_ip6_local_command_fn (vlib_main_t * vm,
2964                            unformat_input_t * input, vlib_cli_command_t * cmd)
2965 {
2966   ip6_main_t *im = &ip6_main;
2967   ip_lookup_main_t *lm = &im->lookup_main;
2968   int i;
2969
2970   vlib_cli_output (vm, "Protocols handled by ip6_local");
2971   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
2972     {
2973       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2974         {
2975
2976           u32 node_index = vlib_get_node (vm,
2977                                           ip6_local_node.index)->
2978             next_nodes[lm->local_next_by_ip_protocol[i]];
2979           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
2980                            node_index);
2981         }
2982     }
2983   return 0;
2984 }
2985
2986
2987
2988 /*?
2989  * Display the set of protocols handled by the local IPv6 stack.
2990  *
2991  * @cliexpar
2992  * Example of how to display local protocol table:
2993  * @cliexstart{show ip6 local}
2994  * Protocols handled by ip6_local
2995  * 17
2996  * 43
2997  * 58
2998  * 115
2999  * @cliexend
3000 ?*/
3001 /* *INDENT-OFF* */
3002 VLIB_CLI_COMMAND (show_ip6_local, static) =
3003 {
3004   .path = "show ip6 local",
3005   .function = show_ip6_local_command_fn,
3006   .short_help = "show ip6 local",
3007 };
3008 /* *INDENT-ON* */
3009
3010 #ifndef CLIB_MARCH_VARIANT
3011 int
3012 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3013                              u32 table_index)
3014 {
3015   vnet_main_t *vnm = vnet_get_main ();
3016   vnet_interface_main_t *im = &vnm->interface_main;
3017   ip6_main_t *ipm = &ip6_main;
3018   ip_lookup_main_t *lm = &ipm->lookup_main;
3019   vnet_classify_main_t *cm = &vnet_classify_main;
3020   ip6_address_t *if_addr;
3021
3022   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3023     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3024
3025   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3026     return VNET_API_ERROR_NO_SUCH_ENTRY;
3027
3028   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3029   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3030
3031   if_addr = ip6_interface_first_address (ipm, sw_if_index);
3032
3033   if (NULL != if_addr)
3034     {
3035       fib_prefix_t pfx = {
3036         .fp_len = 128,
3037         .fp_proto = FIB_PROTOCOL_IP6,
3038         .fp_addr.ip6 = *if_addr,
3039       };
3040       u32 fib_index;
3041
3042       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3043                                                        sw_if_index);
3044
3045
3046       if (table_index != (u32) ~ 0)
3047         {
3048           dpo_id_t dpo = DPO_INVALID;
3049
3050           dpo_set (&dpo,
3051                    DPO_CLASSIFY,
3052                    DPO_PROTO_IP6,
3053                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3054
3055           fib_table_entry_special_dpo_add (fib_index,
3056                                            &pfx,
3057                                            FIB_SOURCE_CLASSIFY,
3058                                            FIB_ENTRY_FLAG_NONE, &dpo);
3059           dpo_reset (&dpo);
3060         }
3061       else
3062         {
3063           fib_table_entry_special_remove (fib_index,
3064                                           &pfx, FIB_SOURCE_CLASSIFY);
3065         }
3066     }
3067
3068   return 0;
3069 }
3070 #endif
3071
3072 static clib_error_t *
3073 set_ip6_classify_command_fn (vlib_main_t * vm,
3074                              unformat_input_t * input,
3075                              vlib_cli_command_t * cmd)
3076 {
3077   u32 table_index = ~0;
3078   int table_index_set = 0;
3079   u32 sw_if_index = ~0;
3080   int rv;
3081
3082   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3083     {
3084       if (unformat (input, "table-index %d", &table_index))
3085         table_index_set = 1;
3086       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3087                          vnet_get_main (), &sw_if_index))
3088         ;
3089       else
3090         break;
3091     }
3092
3093   if (table_index_set == 0)
3094     return clib_error_return (0, "classify table-index must be specified");
3095
3096   if (sw_if_index == ~0)
3097     return clib_error_return (0, "interface / subif must be specified");
3098
3099   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3100
3101   switch (rv)
3102     {
3103     case 0:
3104       break;
3105
3106     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3107       return clib_error_return (0, "No such interface");
3108
3109     case VNET_API_ERROR_NO_SUCH_ENTRY:
3110       return clib_error_return (0, "No such classifier table");
3111     }
3112   return 0;
3113 }
3114
3115 /*?
3116  * Assign a classification table to an interface. The classification
3117  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3118  * commands. Once the table is create, use this command to filter packets
3119  * on an interface.
3120  *
3121  * @cliexpar
3122  * Example of how to assign a classification table to an interface:
3123  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3124 ?*/
3125 /* *INDENT-OFF* */
3126 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3127 {
3128   .path = "set ip6 classify",
3129   .short_help =
3130   "set ip6 classify intfc <interface> table-index <classify-idx>",
3131   .function = set_ip6_classify_command_fn,
3132 };
3133 /* *INDENT-ON* */
3134
3135 static clib_error_t *
3136 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3137 {
3138   ip6_main_t *im = &ip6_main;
3139   uword heapsize = 0;
3140   u32 tmp;
3141   u32 nbuckets = 0;
3142
3143   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3144     {
3145       if (unformat (input, "hash-buckets %d", &tmp))
3146         nbuckets = tmp;
3147       else if (unformat (input, "heap-size %U",
3148                          unformat_memory_size, &heapsize))
3149         ;
3150       else
3151         return clib_error_return (0, "unknown input '%U'",
3152                                   format_unformat_error, input);
3153     }
3154
3155   im->lookup_table_nbuckets = nbuckets;
3156   im->lookup_table_size = heapsize;
3157
3158   return 0;
3159 }
3160
3161 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3162
3163 /*
3164  * fd.io coding-style-patch-verification: ON
3165  *
3166  * Local Variables:
3167  * eval: (c-set-style "gnu")
3168  * End:
3169  */