src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_link.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52
  53 #ifndef CLIB_MARCH_VARIANT
  54 #include <vppinfra/bihash_template.c>
  55 #endif
  56 #include <vnet/ip/ip6_forward.h>
  57 #include <vnet/interface_output.h>
  58
  59 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  60 #define OI_DECAP   0x80000000
  61
  62 static void
  63 ip6_add_interface_prefix_routes (ip6_main_t * im,
  64                                  u32 sw_if_index,
  65                                  u32 fib_index,
  66                                  ip6_address_t * address, u32 address_length)
  67 {
  68   ip_lookup_main_t *lm = &im->lookup_main;
  69   ip_interface_prefix_t *if_prefix;
  70
  71   /* *INDENT-OFF* */
  72   ip_interface_prefix_key_t key = {
  73     .prefix = {
  74       .fp_len = address_length,
  75       .fp_proto = FIB_PROTOCOL_IP6,
  76       .fp_addr.ip6 = {
  77         .as_u64 = {
  78           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
  79           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
  80         },
  81       },
  82     },
  83     .sw_if_index = sw_if_index,
  84   };
  85   /* *INDENT-ON* */
  86
  87   /* If prefix already set on interface, just increment ref count & return */
  88   if_prefix = ip_get_interface_prefix (lm, &key);
  89   if (if_prefix)
  90     {
  91       if_prefix->ref_count += 1;
  92       return;
  93     }
  94
  95   /* New prefix - allocate a pool entry, initialize it, add to the hash */
  96   pool_get (lm->if_prefix_pool, if_prefix);
  97   if_prefix->ref_count = 1;
  98   clib_memcpy (&if_prefix->key, &key, sizeof (key));
  99   mhash_set (&lm->prefix_to_if_prefix_index, &key,
 100              if_prefix - lm->if_prefix_pool, 0 /* old value */ );
 101
 102   /* length < 128 - add glean */
 103   if (address_length < 128)
 104     {
 105       /* set the glean route for the prefix */
 106       fib_table_entry_update_one_path (fib_index, &key.prefix,
 107                                        FIB_SOURCE_INTERFACE,
 108                                        (FIB_ENTRY_FLAG_CONNECTED |
 109                                         FIB_ENTRY_FLAG_ATTACHED),
 110                                        DPO_PROTO_IP6,
 111                                        /* No next-hop address */
 112                                        NULL, sw_if_index,
 113                                        /* invalid FIB index */
 114                                        ~0, 1,
 115                                        /* no out-label stack */
 116                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 117     }
 118 }
 119
 120 static void
 121 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 122                           ip6_main_t * im, u32 fib_index,
 123                           ip_interface_address_t * a)
 124 {
 125   ip_lookup_main_t *lm = &im->lookup_main;
 126   ip6_address_t *address = ip_interface_address_get_address (lm, a);
 127   fib_prefix_t pfx = {
 128     .fp_len = a->address_length,
 129     .fp_proto = FIB_PROTOCOL_IP6,
 130     .fp_addr.ip6 = *address,
 131   };
 132
 133   /* set special routes for the prefix if needed */
 134   ip6_add_interface_prefix_routes (im, sw_if_index, fib_index,
 135                                    address, a->address_length);
 136
 137   pfx.fp_len = 128;
 138   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 139     {
 140       u32 classify_table_index =
 141         lm->classify_table_index_by_sw_if_index[sw_if_index];
 142       if (classify_table_index != (u32) ~ 0)
 143         {
 144           dpo_id_t dpo = DPO_INVALID;
 145
 146           dpo_set (&dpo,
 147                    DPO_CLASSIFY,
 148                    DPO_PROTO_IP6,
 149                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 150
 151           fib_table_entry_special_dpo_add (fib_index,
 152                                            &pfx,
 153                                            FIB_SOURCE_CLASSIFY,
 154                                            FIB_ENTRY_FLAG_NONE, &dpo);
 155           dpo_reset (&dpo);
 156         }
 157     }
 158
 159   fib_table_entry_update_one_path (fib_index, &pfx,
 160                                    FIB_SOURCE_INTERFACE,
 161                                    (FIB_ENTRY_FLAG_CONNECTED |
 162                                     FIB_ENTRY_FLAG_LOCAL),
 163                                    DPO_PROTO_IP6,
 164                                    &pfx.fp_addr,
 165                                    sw_if_index, ~0,
 166                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 167 }
 168
 169 static void
 170 ip6_del_interface_prefix_routes (ip6_main_t * im,
 171                                  u32 sw_if_index,
 172                                  u32 fib_index,
 173                                  ip6_address_t * address, u32 address_length)
 174 {
 175   ip_lookup_main_t *lm = &im->lookup_main;
 176   ip_interface_prefix_t *if_prefix;
 177
 178   /* *INDENT-OFF* */
 179   ip_interface_prefix_key_t key = {
 180     .prefix = {
 181       .fp_len = address_length,
 182       .fp_proto = FIB_PROTOCOL_IP6,
 183       .fp_addr.ip6 = {
 184         .as_u64 = {
 185           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
 186           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
 187         },
 188       },
 189     },
 190     .sw_if_index = sw_if_index,
 191   };
 192   /* *INDENT-ON* */
 193
 194   if_prefix = ip_get_interface_prefix (lm, &key);
 195   if (!if_prefix)
 196     {
 197       clib_warning ("Prefix not found while deleting %U",
 198                     format_ip4_address_and_length, address, address_length);
 199       return;
 200     }
 201
 202   /* If not deleting last intf addr in prefix, decrement ref count & return */
 203   if_prefix->ref_count -= 1;
 204   if (if_prefix->ref_count > 0)
 205     return;
 206
 207   /* length <= 128, delete glean route */
 208   if (address_length <= 128)
 209     {
 210       /* remove glean route for prefix */
 211       fib_table_entry_delete (fib_index, &key.prefix, FIB_SOURCE_INTERFACE);
 212     }
 213
 214   mhash_unset (&lm->prefix_to_if_prefix_index, &key, 0 /* old_value */ );
 215   pool_put (lm->if_prefix_pool, if_prefix);
 216 }
 217
 218 static void
 219 ip6_del_interface_routes (u32 sw_if_index, ip6_main_t * im,
 220                           u32 fib_index,
 221                           ip6_address_t * address, u32 address_length)
 222 {
 223   fib_prefix_t pfx = {
 224     .fp_len = 128,
 225     .fp_proto = FIB_PROTOCOL_IP6,
 226     .fp_addr.ip6 = *address,
 227   };
 228
 229   /* delete special routes for the prefix if needed */
 230   ip6_del_interface_prefix_routes (im, sw_if_index, fib_index,
 231                                    address, address_length);
 232
 233   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 234 }
 235
 236 #ifndef CLIB_MARCH_VARIANT
 237 void
 238 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 239 {
 240   ip6_main_t *im = &ip6_main;
 241
 242   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 243
 244   /*
 245    * enable/disable only on the 1<->0 transition
 246    */
 247   if (is_enable)
 248     {
 249       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 250         return;
 251     }
 252   else
 253     {
 254       /* The ref count is 0 when an address is removed from an interface that has
 255        * no address - this is not a ciritical error */
 256       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 257           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 258         return;
 259     }
 260
 261   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 262                                !is_enable, 0, 0);
 263
 264   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 265                                sw_if_index, !is_enable, 0, 0);
 266 }
 267
 268 /* get first interface address */
 269 ip6_address_t *
 270 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 271 {
 272   ip_lookup_main_t *lm = &im->lookup_main;
 273   ip_interface_address_t *ia = 0;
 274   ip6_address_t *result = 0;
 275
 276   /* *INDENT-OFF* */
 277   foreach_ip_interface_address (lm, ia, sw_if_index,
 278                                 1 /* honor unnumbered */,
 279   ({
 280     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 281     result = a;
 282     break;
 283   }));
 284   /* *INDENT-ON* */
 285   return result;
 286 }
 287
 288 clib_error_t *
 289 ip6_add_del_interface_address (vlib_main_t * vm,
 290                                u32 sw_if_index,
 291                                ip6_address_t * address,
 292                                u32 address_length, u32 is_del)
 293 {
 294   vnet_main_t *vnm = vnet_get_main ();
 295   ip6_main_t *im = &ip6_main;
 296   ip_lookup_main_t *lm = &im->lookup_main;
 297   clib_error_t *error;
 298   u32 if_address_index;
 299   ip6_address_fib_t ip6_af, *addr_fib = 0;
 300   const ip6_address_t *ll_addr;
 301
 302   /* local0 interface doesn't support IP addressing */
 303   if (sw_if_index == 0)
 304     {
 305       return
 306         clib_error_create ("local0 interface doesn't support IP addressing");
 307     }
 308
 309   if (ip6_address_is_link_local_unicast (address))
 310     {
 311       if (address_length != 128)
 312         {
 313           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 314           return
 315             clib_error_create
 316             ("prefix length of link-local address must be 128");
 317         }
 318       if (!is_del)
 319         {
 320           int rv;
 321
 322           rv = ip6_set_link_local_address (sw_if_index, address);
 323
 324           if (rv)
 325             {
 326               vnm->api_errno = rv;
 327               return clib_error_create ("address not assignable");
 328             }
 329         }
 330       else
 331         {
 332           ll_addr = ip6_get_link_local_address (sw_if_index);
 333           if (ip6_address_is_equal (ll_addr, address))
 334             {
 335               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 336               return clib_error_create ("address not deletable");
 337             }
 338           else
 339             {
 340               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 341               return clib_error_create ("address not found");
 342             }
 343         }
 344     }
 345
 346   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 347   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 348
 349   ip6_addr_fib_init (&ip6_af, address,
 350                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 351   vec_add1 (addr_fib, ip6_af);
 352
 353   /* *INDENT-OFF* */
 354   if (!is_del)
 355     {
 356       /* When adding an address check that it does not conflict
 357          with an existing address on any interface in this table. */
 358       ip_interface_address_t *ia;
 359       vnet_sw_interface_t *sif;
 360
 361       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 362       ({
 363           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 364               im->fib_index_by_sw_if_index[sif->sw_if_index])
 365             {
 366               foreach_ip_interface_address
 367                 (&im->lookup_main, ia, sif->sw_if_index,
 368                  0 /* honor unnumbered */ ,
 369                  ({
 370                    ip6_address_t * x =
 371                      ip_interface_address_get_address
 372                      (&im->lookup_main, ia);
 373                    if (ip6_destination_matches_route
 374                        (im, address, x, ia->address_length) ||
 375                        ip6_destination_matches_route (im,
 376                                                       x,
 377                                                       address,
 378                                                       address_length))
 379                      {
 380                        /* an intf may have >1 addr from the same prefix */
 381                        if ((sw_if_index == sif->sw_if_index) &&
 382                            (ia->address_length == address_length) &&
 383                            !ip6_address_is_equal (x, address))
 384                          continue;
 385
 386                        /* error if the length or intf was different */
 387                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 388                        return
 389                          clib_error_create
 390                          ("failed to add %U which conflicts with %U for interface %U",
 391                           format_ip6_address_and_length, address,
 392                           address_length,
 393                           format_ip6_address_and_length, x,
 394                           ia->address_length,
 395                           format_vnet_sw_if_index_name, vnm,
 396                           sif->sw_if_index);
 397                      }
 398                  }));
 399             }
 400       }));
 401     }
 402   /* *INDENT-ON* */
 403
 404   {
 405     uword elts_before = pool_elts (lm->if_address_pool);
 406
 407     error = ip_interface_address_add_del
 408       (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 409     if (error)
 410       goto done;
 411
 412     /* Pool did not grow: add duplicate address. */
 413     if (elts_before == pool_elts (lm->if_address_pool))
 414       goto done;
 415   }
 416
 417   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 418   if (!is_del)
 419     ip6_link_enable (sw_if_index);
 420
 421   /* intf addr routes are added/deleted on admin up/down */
 422   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index))
 423     {
 424       if (is_del)
 425         ip6_del_interface_routes (sw_if_index,
 426                                   im, ip6_af.fib_index, address,
 427                                   address_length);
 428       else
 429         ip6_add_interface_routes (vnm, sw_if_index,
 430                                   im, ip6_af.fib_index,
 431                                   pool_elt_at_index (lm->if_address_pool,
 432                                                      if_address_index));
 433     }
 434   {
 435     ip6_add_del_interface_address_callback_t *cb;
 436     vec_foreach (cb, im->add_del_interface_address_callbacks)
 437       cb->function (im, cb->function_opaque, sw_if_index,
 438                     address, address_length, if_address_index, is_del);
 439   }
 440   if (is_del)
 441     ip6_link_disable (sw_if_index);
 442
 443 done:
 444   vec_free (addr_fib);
 445   return error;
 446 }
 447
 448 #endif
 449
 450 static clib_error_t *
 451 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 452 {
 453   ip6_main_t *im = &ip6_main;
 454   ip_interface_address_t *ia;
 455   ip6_address_t *a;
 456   u32 is_admin_up, fib_index;
 457
 458   /* Fill in lookup tables with default table (0). */
 459   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 460
 461   vec_validate_init_empty (im->
 462                            lookup_main.if_address_pool_index_by_sw_if_index,
 463                            sw_if_index, ~0);
 464
 465   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 466
 467   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 468
 469   /* *INDENT-OFF* */
 470   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 471                                 0 /* honor unnumbered */,
 472   ({
 473     a = ip_interface_address_get_address (&im->lookup_main, ia);
 474     if (is_admin_up)
 475       ip6_add_interface_routes (vnm, sw_if_index,
 476                                 im, fib_index,
 477                                 ia);
 478     else
 479       ip6_del_interface_routes (sw_if_index, im, fib_index,
 480                                 a, ia->address_length);
 481   }));
 482   /* *INDENT-ON* */
 483
 484   return 0;
 485 }
 486
 487 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 488
 489 /* Built-in ip6 unicast rx feature path definition */
 490 /* *INDENT-OFF* */
 491 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 492 {
 493   .arc_name  = "ip6-unicast",
 494   .start_nodes = VNET_FEATURES ("ip6-input"),
 495   .last_in_arc = "ip6-lookup",
 496   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 497 };
 498
 499 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 500 {
 501   .arc_name = "ip6-unicast",
 502   .node_name = "ip6-flow-classify",
 503   .runs_before = VNET_FEATURES ("ip6-inacl"),
 504 };
 505
 506 VNET_FEATURE_INIT (ip6_inacl, static) =
 507 {
 508   .arc_name = "ip6-unicast",
 509   .node_name = "ip6-inacl",
 510   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 511 };
 512
 513 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 514 {
 515   .arc_name = "ip6-unicast",
 516   .node_name = "ip6-policer-classify",
 517   .runs_before = VNET_FEATURES ("ipsec6-input-feature"),
 518 };
 519
 520 VNET_FEATURE_INIT (ip6_ipsec, static) =
 521 {
 522   .arc_name = "ip6-unicast",
 523   .node_name = "ipsec6-input-feature",
 524   .runs_before = VNET_FEATURES ("l2tp-decap"),
 525 };
 526
 527 VNET_FEATURE_INIT (ip6_l2tp, static) =
 528 {
 529   .arc_name = "ip6-unicast",
 530   .node_name = "l2tp-decap",
 531   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 532 };
 533
 534 VNET_FEATURE_INIT (ip6_vpath, static) =
 535 {
 536   .arc_name = "ip6-unicast",
 537   .node_name = "vpath-input-ip6",
 538   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 539 };
 540
 541 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 542 {
 543   .arc_name = "ip6-unicast",
 544   .node_name = "ip6-vxlan-bypass",
 545   .runs_before = VNET_FEATURES ("ip6-lookup"),
 546 };
 547
 548 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 549 {
 550   .arc_name = "ip6-unicast",
 551   .node_name = "ip6-not-enabled",
 552   .runs_before = VNET_FEATURES ("ip6-lookup"),
 553 };
 554
 555 VNET_FEATURE_INIT (ip6_lookup, static) =
 556 {
 557   .arc_name = "ip6-unicast",
 558   .node_name = "ip6-lookup",
 559   .runs_before = 0,  /*last feature*/
 560 };
 561
 562 /* Built-in ip6 multicast rx feature path definition (none now) */
 563 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 564 {
 565   .arc_name  = "ip6-multicast",
 566   .start_nodes = VNET_FEATURES ("ip6-input"),
 567   .last_in_arc = "ip6-mfib-forward-lookup",
 568   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 569 };
 570
 571 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 572   .arc_name = "ip6-multicast",
 573   .node_name = "vpath-input-ip6",
 574   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 575 };
 576
 577 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 578   .arc_name = "ip6-multicast",
 579   .node_name = "ip6-not-enabled",
 580   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 581 };
 582
 583 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 584   .arc_name = "ip6-multicast",
 585   .node_name = "ip6-mfib-forward-lookup",
 586   .runs_before = 0, /* last feature */
 587 };
 588
 589 /* Built-in ip4 tx feature path definition */
 590 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 591 {
 592   .arc_name  = "ip6-output",
 593   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 594   .last_in_arc = "interface-output",
 595   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 596 };
 597
 598 VNET_FEATURE_INIT (ip6_outacl, static) = {
 599   .arc_name = "ip6-output",
 600   .node_name = "ip6-outacl",
 601   .runs_before = VNET_FEATURES ("ipsec6-output-feature"),
 602 };
 603
 604 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 605   .arc_name = "ip6-output",
 606   .node_name = "ipsec6-output-feature",
 607   .runs_before = VNET_FEATURES ("interface-output"),
 608 };
 609
 610 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 611   .arc_name = "ip6-output",
 612   .node_name = "interface-output",
 613   .runs_before = 0, /* not before any other features */
 614 };
 615 /* *INDENT-ON* */
 616
 617 static clib_error_t *
 618 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 619 {
 620   ip6_main_t *im = &ip6_main;
 621
 622   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 623   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 624
 625   if (!is_add)
 626     {
 627       /* Ensure that IPv6 is disabled */
 628       ip6_main_t *im6 = &ip6_main;
 629       ip_lookup_main_t *lm6 = &im6->lookup_main;
 630       ip_interface_address_t *ia = 0;
 631       ip6_address_t *address;
 632       vlib_main_t *vm = vlib_get_main ();
 633
 634       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 635       /* *INDENT-OFF* */
 636       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 637       ({
 638         address = ip_interface_address_get_address (lm6, ia);
 639         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 640       }));
 641       /* *INDENT-ON* */
 642       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 643     }
 644
 645   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 646                                is_add, 0, 0);
 647
 648   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 649                                sw_if_index, is_add, 0, 0);
 650
 651   return /* no error */ 0;
 652 }
 653
 654 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 655
 656 VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 657                                 vlib_node_runtime_t * node,
 658                                 vlib_frame_t * frame)
 659 {
 660   return ip6_lookup_inline (vm, node, frame);
 661 }
 662
 663 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 664
 665 /* *INDENT-OFF* */
 666 VLIB_REGISTER_NODE (ip6_lookup_node) =
 667 {
 668   .name = "ip6-lookup",
 669   .vector_size = sizeof (u32),
 670   .format_trace = format_ip6_lookup_trace,
 671   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 672   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 673 };
 674 /* *INDENT-ON* */
 675
 676 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
 677                                       vlib_node_runtime_t * node,
 678                                       vlib_frame_t * frame)
 679 {
 680   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 681   u32 n_left, *from;
 682   u32 thread_index = vm->thread_index;
 683   ip6_main_t *im = &ip6_main;
 684   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 685   u16 nexts[VLIB_FRAME_SIZE], *next;
 686
 687   from = vlib_frame_vector_args (frame);
 688   n_left = frame->n_vectors;
 689   next = nexts;
 690
 691   vlib_get_buffers (vm, from, bufs, n_left);
 692
 693   while (n_left >= 4)
 694     {
 695       const load_balance_t *lb0, *lb1;
 696       const ip6_header_t *ip0, *ip1;
 697       u32 lbi0, hc0, lbi1, hc1;
 698       const dpo_id_t *dpo0, *dpo1;
 699
 700       /* Prefetch next iteration. */
 701       {
 702         vlib_prefetch_buffer_header (b[2], STORE);
 703         vlib_prefetch_buffer_header (b[3], STORE);
 704
 705         CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), STORE);
 706         CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), STORE);
 707       }
 708
 709       ip0 = vlib_buffer_get_current (b[0]);
 710       ip1 = vlib_buffer_get_current (b[1]);
 711       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 712       lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
 713
 714       lb0 = load_balance_get (lbi0);
 715       lb1 = load_balance_get (lbi1);
 716
 717       /*
 718        * this node is for via FIBs we can re-use the hash value from the
 719        * to node if present.
 720        * We don't want to use the same hash value at each level in the recursion
 721        * graph as that would lead to polarisation
 722        */
 723       hc0 = hc1 = 0;
 724
 725       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 726         {
 727           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 728             {
 729               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 730                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 731             }
 732           else
 733             {
 734               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 735                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 736             }
 737           dpo0 = load_balance_get_fwd_bucket
 738             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 739         }
 740       else
 741         {
 742           dpo0 = load_balance_get_bucket_i (lb0, 0);
 743         }
 744       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 745         {
 746           if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash))
 747             {
 748               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 749                 vnet_buffer (b[1])->ip.flow_hash >> 1;
 750             }
 751           else
 752             {
 753               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 754                 ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 755             }
 756           dpo1 = load_balance_get_fwd_bucket
 757             (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 758         }
 759       else
 760         {
 761           dpo1 = load_balance_get_bucket_i (lb1, 0);
 762         }
 763
 764       next[0] = dpo0->dpoi_next_node;
 765       next[1] = dpo1->dpoi_next_node;
 766
 767       /* Only process the HBH Option Header if explicitly configured to do so */
 768       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 769         {
 770           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 771             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 772         }
 773       /* Only process the HBH Option Header if explicitly configured to do so */
 774       if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 775         {
 776           next[1] = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 777             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[1];
 778         }
 779
 780       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 781       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 782
 783       vlib_increment_combined_counter
 784         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 785       vlib_increment_combined_counter
 786         (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1]));
 787
 788       b += 2;
 789       next += 2;
 790       n_left -= 2;
 791     }
 792
 793   while (n_left > 0)
 794     {
 795       const load_balance_t *lb0;
 796       const ip6_header_t *ip0;
 797       const dpo_id_t *dpo0;
 798       u32 lbi0, hc0;
 799
 800       ip0 = vlib_buffer_get_current (b[0]);
 801       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 802
 803       lb0 = load_balance_get (lbi0);
 804
 805       hc0 = 0;
 806       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 807         {
 808           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 809             {
 810               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 811                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 812             }
 813           else
 814             {
 815               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 816                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 817             }
 818           dpo0 = load_balance_get_fwd_bucket
 819             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 820         }
 821       else
 822         {
 823           dpo0 = load_balance_get_bucket_i (lb0, 0);
 824         }
 825
 826       next[0] = dpo0->dpoi_next_node;
 827       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 828
 829       /* Only process the HBH Option Header if explicitly configured to do so */
 830       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 831         {
 832           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 833             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 834         }
 835
 836       vlib_increment_combined_counter
 837         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 838
 839       b += 1;
 840       next += 1;
 841       n_left -= 1;
 842     }
 843
 844   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
 845
 846   if (node->flags & VLIB_NODE_FLAG_TRACE)
 847     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 848
 849   return frame->n_vectors;
 850 }
 851
 852 /* *INDENT-OFF* */
 853 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 854 {
 855   .name = "ip6-load-balance",
 856   .vector_size = sizeof (u32),
 857   .sibling_of = "ip6-lookup",
 858   .format_trace = format_ip6_lookup_trace,
 859 };
 860 /* *INDENT-ON* */
 861
 862 typedef struct
 863 {
 864   /* Adjacency taken. */
 865   u32 adj_index;
 866   u32 flow_hash;
 867   u32 fib_index;
 868
 869   /* Packet data, possibly *after* rewrite. */
 870   u8 packet_data[128 - 1 * sizeof (u32)];
 871 }
 872 ip6_forward_next_trace_t;
 873
 874 #ifndef CLIB_MARCH_VARIANT
 875 u8 *
 876 format_ip6_forward_next_trace (u8 * s, va_list * args)
 877 {
 878   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 879   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 880   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 881   u32 indent = format_get_indent (s);
 882
 883   s = format (s, "%U%U",
 884               format_white_space, indent,
 885               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 886   return s;
 887 }
 888 #endif
 889
 890 static u8 *
 891 format_ip6_lookup_trace (u8 * s, va_list * args)
 892 {
 893   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 894   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 895   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 896   u32 indent = format_get_indent (s);
 897
 898   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 899               t->fib_index, t->adj_index, t->flow_hash);
 900   s = format (s, "\n%U%U",
 901               format_white_space, indent,
 902               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 903   return s;
 904 }
 905
 906
 907 static u8 *
 908 format_ip6_rewrite_trace (u8 * s, va_list * args)
 909 {
 910   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 911   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 912   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 913   u32 indent = format_get_indent (s);
 914
 915   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 916               t->fib_index, t->adj_index, format_ip_adjacency,
 917               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 918   s = format (s, "\n%U%U",
 919               format_white_space, indent,
 920               format_ip_adjacency_packet_data,
 921               t->packet_data, sizeof (t->packet_data));
 922   return s;
 923 }
 924
 925 /* Common trace function for all ip6-forward next nodes. */
 926 #ifndef CLIB_MARCH_VARIANT
 927 void
 928 ip6_forward_next_trace (vlib_main_t * vm,
 929                         vlib_node_runtime_t * node,
 930                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 931 {
 932   u32 *from, n_left;
 933   ip6_main_t *im = &ip6_main;
 934
 935   n_left = frame->n_vectors;
 936   from = vlib_frame_vector_args (frame);
 937
 938   while (n_left >= 4)
 939     {
 940       u32 bi0, bi1;
 941       vlib_buffer_t *b0, *b1;
 942       ip6_forward_next_trace_t *t0, *t1;
 943
 944       /* Prefetch next iteration. */
 945       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 946       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 947
 948       bi0 = from[0];
 949       bi1 = from[1];
 950
 951       b0 = vlib_get_buffer (vm, bi0);
 952       b1 = vlib_get_buffer (vm, bi1);
 953
 954       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 955         {
 956           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 957           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 958           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 959           t0->fib_index =
 960             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 961              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 962             vec_elt (im->fib_index_by_sw_if_index,
 963                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 964
 965           clib_memcpy_fast (t0->packet_data,
 966                             vlib_buffer_get_current (b0),
 967                             sizeof (t0->packet_data));
 968         }
 969       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 970         {
 971           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 972           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 973           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 974           t1->fib_index =
 975             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
 976              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
 977             vec_elt (im->fib_index_by_sw_if_index,
 978                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
 979
 980           clib_memcpy_fast (t1->packet_data,
 981                             vlib_buffer_get_current (b1),
 982                             sizeof (t1->packet_data));
 983         }
 984       from += 2;
 985       n_left -= 2;
 986     }
 987
 988   while (n_left >= 1)
 989     {
 990       u32 bi0;
 991       vlib_buffer_t *b0;
 992       ip6_forward_next_trace_t *t0;
 993
 994       bi0 = from[0];
 995
 996       b0 = vlib_get_buffer (vm, bi0);
 997
 998       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 999         {
1000           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1001           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1002           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1003           t0->fib_index =
1004             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1005              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1006             vec_elt (im->fib_index_by_sw_if_index,
1007                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1008
1009           clib_memcpy_fast (t0->packet_data,
1010                             vlib_buffer_get_current (b0),
1011                             sizeof (t0->packet_data));
1012         }
1013       from += 1;
1014       n_left -= 1;
1015     }
1016 }
1017
1018 /* Compute TCP/UDP/ICMP6 checksum in software. */
1019 u16
1020 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1021                                    ip6_header_t * ip0, int *bogus_lengthp)
1022 {
1023   ip_csum_t sum0;
1024   u16 payload_length_host_byte_order;
1025   u32 i;
1026   u32 headers_size = sizeof (ip0[0]);
1027   u8 *data_this_buffer;
1028
1029   ASSERT (bogus_lengthp);
1030   *bogus_lengthp = 0;
1031
1032   /* Initialize checksum with ip header. */
1033   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1034   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1035   data_this_buffer = (u8 *) (ip0 + 1);
1036
1037   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1038     {
1039       sum0 = ip_csum_with_carry
1040         (sum0, clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1041       sum0 = ip_csum_with_carry
1042         (sum0, clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1043     }
1044
1045   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
1046    * or UDP-Ping packets */
1047   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1048     {
1049       u32 skip_bytes;
1050       ip6_hop_by_hop_ext_t *ext_hdr =
1051         (ip6_hop_by_hop_ext_t *) data_this_buffer;
1052
1053       /* validate really icmp6 next */
1054       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1055               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
1056
1057       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
1058       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
1059
1060       payload_length_host_byte_order -= skip_bytes;
1061       headers_size += skip_bytes;
1062     }
1063
1064   if (p0)
1065     return ip_calculate_l4_checksum (vm, p0, sum0,
1066                                      payload_length_host_byte_order,
1067                                      (u8 *) ip0, headers_size, NULL);
1068   else
1069     return ip_calculate_l4_checksum (vm, 0, sum0,
1070                                      payload_length_host_byte_order, NULL, 0,
1071                                      data_this_buffer);
1072 }
1073
1074 u32
1075 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1076 {
1077   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1078   udp_header_t *udp0;
1079   u16 sum16;
1080   int bogus_length;
1081
1082   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1083   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1084           || ip0->protocol == IP_PROTOCOL_ICMP6
1085           || ip0->protocol == IP_PROTOCOL_UDP
1086           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1087
1088   udp0 = (void *) (ip0 + 1);
1089   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1090     {
1091       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1092                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1093       return p0->flags;
1094     }
1095
1096   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1097
1098   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1099                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1100
1101   return p0->flags;
1102 }
1103 #endif
1104
1105 /**
1106  * @brief returns number of links on which src is reachable.
1107  */
1108 always_inline int
1109 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1110 {
1111   const load_balance_t *lb0;
1112   index_t lbi;
1113   u32 fib_index;
1114
1115   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1116                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1117   fib_index =
1118     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1119     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1120
1121   lbi = ip6_fib_table_fwding_lookup (fib_index, &i->src_address);
1122   lb0 = load_balance_get (lbi);
1123
1124   return (fib_urpf_check_size (lb0->lb_urpf));
1125 }
1126
1127 always_inline u8
1128 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1129                            u32 * udp_offset0)
1130 {
1131   u32 proto0;
1132   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1133   if (proto0 != IP_PROTOCOL_UDP)
1134     {
1135       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1136       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1137     }
1138   return proto0;
1139 }
1140
1141 /* *INDENT-OFF* */
1142 VNET_FEATURE_ARC_INIT (ip6_local) =
1143 {
1144   .arc_name  = "ip6-local",
1145   .start_nodes = VNET_FEATURES ("ip6-local"),
1146 };
1147 /* *INDENT-ON* */
1148
1149 static_always_inline u8
1150 ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
1151 {
1152
1153   u16 payload_length_host_byte_order;
1154   u32 n_this_buffer, n_bytes_left;
1155   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1156   u32 headers_size = sizeof (ip0[0]);
1157   u8 *data_this_buffer;
1158
1159
1160   data_this_buffer = (u8 *) (ip0 + 1);
1161
1162   ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t *) data_this_buffer;
1163
1164   /* validate really icmp6 next */
1165
1166   if (!(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1167       || (ext_hdr->next_hdr == IP_PROTOCOL_UDP))
1168     return 0;
1169
1170
1171   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1172   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1173
1174
1175   u32 n_ip_bytes_this_buffer =
1176     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1177   if (n_this_buffer + headers_size > n_ip_bytes_this_buffer)
1178     {
1179       n_this_buffer = p0->current_length > headers_size ?
1180         n_ip_bytes_this_buffer - headers_size : 0;
1181     }
1182
1183   n_bytes_left -= n_this_buffer;
1184   n_bytes_left -= p0->total_length_not_including_first_buffer;
1185
1186   if (n_bytes_left == 0)
1187     return 0;
1188   else
1189     return 1;
1190 }
1191
1192
1193 always_inline uword
1194 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1195                   vlib_frame_t * frame, int head_of_feature_arc)
1196 {
1197   ip6_main_t *im = &ip6_main;
1198   ip_lookup_main_t *lm = &im->lookup_main;
1199   u32 *from, n_left_from;
1200   vlib_node_runtime_t *error_node =
1201     vlib_node_get_runtime (vm, ip6_input_node.index);
1202   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1203   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1204   u16 nexts[VLIB_FRAME_SIZE], *next;
1205
1206   from = vlib_frame_vector_args (frame);
1207   n_left_from = frame->n_vectors;
1208
1209   if (node->flags & VLIB_NODE_FLAG_TRACE)
1210     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1211
1212   vlib_get_buffers (vm, from, bufs, n_left_from);
1213   b = bufs;
1214   next = nexts;
1215
1216   while (n_left_from > 2)
1217     {
1218       /* Prefetch next iteration. */
1219       if (n_left_from >= 6)
1220         {
1221           vlib_prefetch_buffer_header (b[4], STORE);
1222           vlib_prefetch_buffer_header (b[5], STORE);
1223           vlib_prefetch_buffer_data (b[2], LOAD);
1224           vlib_prefetch_buffer_data (b[3], LOAD);
1225         }
1226
1227       u8 error[2];
1228       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1229       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1230
1231       ip6_header_t *ip[2];
1232       ip[0] = vlib_buffer_get_current (b[0]);
1233       ip[1] = vlib_buffer_get_current (b[1]);
1234
1235       if (head_of_feature_arc)
1236         {
1237           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1238           vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1239
1240           u8 type[2];
1241           type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol];
1242           type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol];
1243
1244           u32 flags[2];
1245           flags[0] = b[0]->flags;
1246           flags[1] = b[1]->flags;
1247
1248           u32 good_l4_csum[2];
1249           good_l4_csum[0] =
1250             flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1251                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1252                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1253           good_l4_csum[1] =
1254             flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1255                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1256                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1257
1258           u32 udp_offset[2] = { };
1259           u8 is_tcp_udp[2];
1260           is_tcp_udp[0] =
1261             ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]);
1262           is_tcp_udp[1] =
1263             ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]);
1264           i16 len_diff[2] = { 0 };
1265           if (PREDICT_TRUE (is_tcp_udp[0]))
1266             {
1267               udp_header_t *udp =
1268                 (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]);
1269               good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP
1270                 && udp->checksum == 0;
1271               /* optimistically verify UDP length. */
1272               u16 ip_len, udp_len;
1273               ip_len = clib_net_to_host_u16 (ip[0]->payload_length);
1274               udp_len = clib_net_to_host_u16 (udp->length);
1275               len_diff[0] = ip_len - udp_len;
1276             }
1277           if (PREDICT_TRUE (is_tcp_udp[1]))
1278             {
1279               udp_header_t *udp =
1280                 (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]);
1281               good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP
1282                 && udp->checksum == 0;
1283               /* optimistically verify UDP length. */
1284               u16 ip_len, udp_len;
1285               ip_len = clib_net_to_host_u16 (ip[1]->payload_length);
1286               udp_len = clib_net_to_host_u16 (udp->length);
1287               len_diff[1] = ip_len - udp_len;
1288             }
1289
1290           good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1291           good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1292
1293           len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0;
1294           len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0;
1295
1296           u8 need_csum[2];
1297           need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN
1298             && !good_l4_csum[0]
1299             && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1300           need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN
1301             && !good_l4_csum[1]
1302             && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1303           if (PREDICT_FALSE (need_csum[0]))
1304             {
1305               flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1306               good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1307               error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1308             }
1309           else
1310             {
1311               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1312                 error[0] = IP6_ERROR_BAD_LENGTH;
1313             }
1314           if (PREDICT_FALSE (need_csum[1]))
1315             {
1316               flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]);
1317               good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1318               error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1319             }
1320           else
1321             {
1322               if (ip6_tcp_udp_icmp_bad_length (vm, b[1]))
1323                 error[1] = IP6_ERROR_BAD_LENGTH;
1324             }
1325
1326
1327           error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0];
1328
1329           error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1];
1330
1331           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1332                          IP6_ERROR_UDP_CHECKSUM,
1333                          "Wrong IP6 errors constants");
1334           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1335                          IP6_ERROR_ICMP_CHECKSUM,
1336                          "Wrong IP6 errors constants");
1337
1338           error[0] =
1339             !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0];
1340           error[1] =
1341             !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1];
1342
1343           /* Drop packets from unroutable hosts. */
1344           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1345           u8 unroutable[2];
1346           unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL
1347             && type[0] != IP_BUILTIN_PROTOCOL_ICMP
1348             && !ip6_address_is_link_local_unicast (&ip[0]->src_address);
1349           unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL
1350             && type[1] != IP_BUILTIN_PROTOCOL_ICMP
1351             && !ip6_address_is_link_local_unicast (&ip[1]->src_address);
1352           if (PREDICT_FALSE (unroutable[0]))
1353             {
1354               error[0] =
1355                 !ip6_urpf_loose_check (im, b[0],
1356                                        ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS
1357                 : error[0];
1358             }
1359           if (PREDICT_FALSE (unroutable[1]))
1360             {
1361               error[1] =
1362                 !ip6_urpf_loose_check (im, b[1],
1363                                        ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS
1364                 : error[1];
1365             }
1366
1367           vnet_buffer (b[0])->ip.fib_index =
1368             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1369             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1370             vnet_buffer (b[0])->ip.fib_index;
1371           vnet_buffer (b[1])->ip.fib_index =
1372             vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1373             vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1374             vnet_buffer (b[1])->ip.fib_index;
1375         }                       /* head_of_feature_arc */
1376
1377       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
1378       next[0] =
1379         error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1380       next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol];
1381       next[1] =
1382         error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1];
1383
1384       b[0]->error = error_node->errors[0];
1385       b[1]->error = error_node->errors[1];
1386
1387       if (head_of_feature_arc)
1388         {
1389           u8 ip6_unknown[2];
1390           ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1391           ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1392           if (PREDICT_TRUE (ip6_unknown[0]))
1393             {
1394               u32 next32 = next[0];
1395               vnet_feature_arc_start (arc_index,
1396                                       vnet_buffer (b[0])->sw_if_index
1397                                       [VLIB_RX], &next32, b[0]);
1398               next[0] = next32;
1399             }
1400           if (PREDICT_TRUE (ip6_unknown[1]))
1401             {
1402               u32 next32 = next[1];
1403               vnet_feature_arc_start (arc_index,
1404                                       vnet_buffer (b[1])->sw_if_index
1405                                       [VLIB_RX], &next32, b[1]);
1406               next[1] = next32;
1407             }
1408         }
1409
1410       /* next */
1411       b += 2;
1412       next += 2;
1413       n_left_from -= 2;
1414     }
1415
1416   while (n_left_from)
1417     {
1418       u8 error;
1419       error = IP6_ERROR_UNKNOWN_PROTOCOL;
1420
1421       ip6_header_t *ip;
1422       ip = vlib_buffer_get_current (b[0]);
1423
1424       if (head_of_feature_arc)
1425         {
1426           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1427           u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
1428
1429           u32 flags = b[0]->flags;
1430           u32 good_l4_csum =
1431             flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1432                      VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1433                      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1434
1435           u32 udp_offset;
1436           i16 len_diff = 0;
1437           u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
1438           if (PREDICT_TRUE (is_tcp_udp))
1439             {
1440               udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset);
1441               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1442               good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP
1443                 && udp->checksum == 0;
1444               /* optimistically verify UDP length. */
1445               u16 ip_len, udp_len;
1446               ip_len = clib_net_to_host_u16 (ip->payload_length);
1447               udp_len = clib_net_to_host_u16 (udp->length);
1448               len_diff = ip_len - udp_len;
1449             }
1450
1451           good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
1452           len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
1453
1454           u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
1455             && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1456           if (PREDICT_FALSE (need_csum))
1457             {
1458               flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1459               good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1460               error = IP6_ERROR_UNKNOWN_PROTOCOL;
1461             }
1462           else
1463             {
1464               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1465                 error = IP6_ERROR_BAD_LENGTH;
1466             }
1467
1468
1469
1470           error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
1471
1472           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1473                          IP6_ERROR_UDP_CHECKSUM,
1474                          "Wrong IP6 errors constants");
1475           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1476                          IP6_ERROR_ICMP_CHECKSUM,
1477                          "Wrong IP6 errors constants");
1478
1479           error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error;
1480
1481           /* Drop packets from unroutable hosts. */
1482           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1483           u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL
1484             && type != IP_BUILTIN_PROTOCOL_ICMP
1485             && !ip6_address_is_link_local_unicast (&ip->src_address);
1486           if (PREDICT_FALSE (unroutable))
1487             {
1488               error =
1489                 !ip6_urpf_loose_check (im, b[0],
1490                                        ip) ? IP6_ERROR_SRC_LOOKUP_MISS :
1491                 error;
1492             }
1493
1494           vnet_buffer (b[0])->ip.fib_index =
1495             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1496             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1497             vnet_buffer (b[0])->ip.fib_index;
1498         }                       /* head_of_feature_arc */
1499
1500       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
1501       next[0] =
1502         error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1503
1504       b[0]->error = error_node->errors[0];
1505
1506       if (head_of_feature_arc)
1507         {
1508           if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1509             {
1510               u32 next32 = next[0];
1511               vnet_feature_arc_start (arc_index,
1512                                       vnet_buffer (b[0])->sw_if_index
1513                                       [VLIB_RX], &next32, b[0]);
1514               next[0] = next32;
1515             }
1516         }
1517
1518       /* next */
1519       b += 1;
1520       next += 1;
1521       n_left_from -= 1;
1522     }
1523
1524   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1525   return frame->n_vectors;
1526 }
1527
1528 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1529                                vlib_frame_t * frame)
1530 {
1531   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1532 }
1533
1534 /* *INDENT-OFF* */
1535 VLIB_REGISTER_NODE (ip6_local_node) =
1536 {
1537   .name = "ip6-local",
1538   .vector_size = sizeof (u32),
1539   .format_trace = format_ip6_forward_next_trace,
1540   .n_next_nodes = IP_LOCAL_N_NEXT,
1541   .next_nodes =
1542   {
1543     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1544     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1545     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1546     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1547     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-full-reassembly",
1548   },
1549 };
1550 /* *INDENT-ON* */
1551
1552 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
1553                                           vlib_node_runtime_t * node,
1554                                           vlib_frame_t * frame)
1555 {
1556   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1557 }
1558
1559 /* *INDENT-OFF* */
1560 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
1561   .name = "ip6-local-end-of-arc",
1562   .vector_size = sizeof (u32),
1563
1564   .format_trace = format_ip6_forward_next_trace,
1565   .sibling_of = "ip6-local",
1566 };
1567
1568 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1569   .arc_name = "ip6-local",
1570   .node_name = "ip6-local-end-of-arc",
1571   .runs_before = 0, /* not before any other features */
1572 };
1573 /* *INDENT-ON* */
1574
1575 #ifdef CLIB_MARCH_VARIANT
1576 extern vlib_node_registration_t ip6_local_node;
1577 #else
1578 void
1579 ip6_register_protocol (u32 protocol, u32 node_index)
1580 {
1581   vlib_main_t *vm = vlib_get_main ();
1582   ip6_main_t *im = &ip6_main;
1583   ip_lookup_main_t *lm = &im->lookup_main;
1584
1585   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1586   lm->local_next_by_ip_protocol[protocol] =
1587     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1588 }
1589
1590 void
1591 ip6_unregister_protocol (u32 protocol)
1592 {
1593   ip6_main_t *im = &ip6_main;
1594   ip_lookup_main_t *lm = &im->lookup_main;
1595
1596   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1597   lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT;
1598 }
1599 #endif
1600
1601 typedef enum
1602 {
1603   IP6_REWRITE_NEXT_DROP,
1604   IP6_REWRITE_NEXT_ICMP_ERROR,
1605   IP6_REWRITE_NEXT_FRAGMENT,
1606   IP6_REWRITE_N_NEXT            /* Last */
1607 } ip6_rewrite_next_t;
1608
1609 /**
1610  * This bits of an IPv6 address to mask to construct a multicast
1611  * MAC address
1612  */
1613 #define IP6_MCAST_ADDR_MASK 0xffffffff
1614
1615 always_inline void
1616 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1617                u16 adj_packet_bytes, bool is_locally_generated,
1618                u32 * next, u8 is_midchain, u32 * error)
1619 {
1620   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1621     {
1622       if (is_locally_generated)
1623         {
1624           /* IP fragmentation */
1625           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1626                                    (is_midchain ?
1627                                     IP_FRAG_NEXT_IP_REWRITE_MIDCHAIN :
1628                                     IP_FRAG_NEXT_IP_REWRITE), 0);
1629           *next = IP6_REWRITE_NEXT_FRAGMENT;
1630           *error = IP6_ERROR_MTU_EXCEEDED;
1631         }
1632       else
1633         {
1634           *error = IP6_ERROR_MTU_EXCEEDED;
1635           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1636                                        adj_packet_bytes);
1637           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1638         }
1639     }
1640 }
1641
1642 always_inline uword
1643 ip6_rewrite_inline_with_gso (vlib_main_t * vm,
1644                              vlib_node_runtime_t * node,
1645                              vlib_frame_t * frame,
1646                              int do_counters, int is_midchain, int is_mcast)
1647 {
1648   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1649   u32 *from = vlib_frame_vector_args (frame);
1650   u32 n_left_from, n_left_to_next, *to_next, next_index;
1651   vlib_node_runtime_t *error_node =
1652     vlib_node_get_runtime (vm, ip6_input_node.index);
1653
1654   n_left_from = frame->n_vectors;
1655   next_index = node->cached_next_index;
1656   u32 thread_index = vm->thread_index;
1657
1658   while (n_left_from > 0)
1659     {
1660       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1661
1662       while (n_left_from >= 4 && n_left_to_next >= 2)
1663         {
1664           const ip_adjacency_t *adj0, *adj1;
1665           vlib_buffer_t *p0, *p1;
1666           ip6_header_t *ip0, *ip1;
1667           u32 pi0, rw_len0, next0, error0, adj_index0;
1668           u32 pi1, rw_len1, next1, error1, adj_index1;
1669           u32 tx_sw_if_index0, tx_sw_if_index1;
1670           bool is_locally_originated0, is_locally_originated1;
1671
1672           /* Prefetch next iteration. */
1673           {
1674             vlib_buffer_t *p2, *p3;
1675
1676             p2 = vlib_get_buffer (vm, from[2]);
1677             p3 = vlib_get_buffer (vm, from[3]);
1678
1679             vlib_prefetch_buffer_header (p2, LOAD);
1680             vlib_prefetch_buffer_header (p3, LOAD);
1681
1682             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1683             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1684
1685             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1686             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1687           }
1688
1689           pi0 = to_next[0] = from[0];
1690           pi1 = to_next[1] = from[1];
1691
1692           from += 2;
1693           n_left_from -= 2;
1694           to_next += 2;
1695           n_left_to_next -= 2;
1696
1697           p0 = vlib_get_buffer (vm, pi0);
1698           p1 = vlib_get_buffer (vm, pi1);
1699
1700           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1701           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1702
1703           ip0 = vlib_buffer_get_current (p0);
1704           ip1 = vlib_buffer_get_current (p1);
1705
1706           error0 = error1 = IP6_ERROR_NONE;
1707           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1708
1709           is_locally_originated0 =
1710             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1711           if (PREDICT_TRUE (!is_locally_originated0))
1712             {
1713               i32 hop_limit0 = ip0->hop_limit;
1714
1715               /* Input node should have reject packets with hop limit 0. */
1716               ASSERT (ip0->hop_limit > 0);
1717
1718               hop_limit0 -= 1;
1719
1720               ip0->hop_limit = hop_limit0;
1721
1722               /*
1723                * If the hop count drops below 1 when forwarding, generate
1724                * an ICMP response.
1725                */
1726               if (PREDICT_FALSE (hop_limit0 <= 0))
1727                 {
1728                   error0 = IP6_ERROR_TIME_EXPIRED;
1729                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1730                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1731                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1732                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1733                                                0);
1734                 }
1735             }
1736           else
1737             {
1738               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1739             }
1740           is_locally_originated1 =
1741             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1742           if (PREDICT_TRUE (!is_locally_originated1))
1743             {
1744               i32 hop_limit1 = ip1->hop_limit;
1745
1746               /* Input node should have reject packets with hop limit 0. */
1747               ASSERT (ip1->hop_limit > 0);
1748
1749               hop_limit1 -= 1;
1750
1751               ip1->hop_limit = hop_limit1;
1752
1753               /*
1754                * If the hop count drops below 1 when forwarding, generate
1755                * an ICMP response.
1756                */
1757               if (PREDICT_FALSE (hop_limit1 <= 0))
1758                 {
1759                   error1 = IP6_ERROR_TIME_EXPIRED;
1760                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1761                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1762                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1763                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1764                                                0);
1765                 }
1766             }
1767           else
1768             {
1769               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1770             }
1771           adj0 = adj_get (adj_index0);
1772           adj1 = adj_get (adj_index1);
1773
1774           rw_len0 = adj0[0].rewrite_header.data_bytes;
1775           rw_len1 = adj1[0].rewrite_header.data_bytes;
1776           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1777           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1778
1779           if (do_counters)
1780             {
1781               vlib_increment_combined_counter
1782                 (&adjacency_counters,
1783                  thread_index, adj_index0, 1,
1784                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1785               vlib_increment_combined_counter
1786                 (&adjacency_counters,
1787                  thread_index, adj_index1, 1,
1788                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1789             }
1790
1791           /* Check MTU of outgoing interface. */
1792           u16 ip0_len =
1793             clib_net_to_host_u16 (ip0->payload_length) +
1794             sizeof (ip6_header_t);
1795           u16 ip1_len =
1796             clib_net_to_host_u16 (ip1->payload_length) +
1797             sizeof (ip6_header_t);
1798           if (p0->flags & VNET_BUFFER_F_GSO)
1799             ip0_len = gso_mtu_sz (p0);
1800           if (p1->flags & VNET_BUFFER_F_GSO)
1801             ip1_len = gso_mtu_sz (p1);
1802
1803
1804
1805           ip6_mtu_check (p0, ip0_len,
1806                          adj0[0].rewrite_header.max_l3_packet_bytes,
1807                          is_locally_originated0, &next0, is_midchain,
1808                          &error0);
1809           ip6_mtu_check (p1, ip1_len,
1810                          adj1[0].rewrite_header.max_l3_packet_bytes,
1811                          is_locally_originated1, &next1, is_midchain,
1812                          &error1);
1813
1814           /* Don't adjust the buffer for hop count issue; icmp-error node
1815            * wants to see the IP header */
1816           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1817             {
1818               p0->current_data -= rw_len0;
1819               p0->current_length += rw_len0;
1820
1821               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1822               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1823               next0 = adj0[0].rewrite_header.next_index;
1824
1825               if (PREDICT_FALSE
1826                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1827                 vnet_feature_arc_start (lm->output_feature_arc_index,
1828                                         tx_sw_if_index0, &next0, p0);
1829             }
1830           else
1831             {
1832               p0->error = error_node->errors[error0];
1833             }
1834           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1835             {
1836               p1->current_data -= rw_len1;
1837               p1->current_length += rw_len1;
1838
1839               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1840               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1841               next1 = adj1[0].rewrite_header.next_index;
1842
1843               if (PREDICT_FALSE
1844                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1845                 vnet_feature_arc_start (lm->output_feature_arc_index,
1846                                         tx_sw_if_index1, &next1, p1);
1847             }
1848           else
1849             {
1850               p1->error = error_node->errors[error1];
1851             }
1852
1853           if (is_midchain)
1854             {
1855               /* before we paint on the next header, update the L4
1856                * checksums if required, since there's no offload on a tunnel */
1857               calc_checksums (vm, p0);
1858               calc_checksums (vm, p1);
1859             }
1860
1861           /* Guess we are only writing on simple Ethernet header. */
1862           vnet_rewrite_two_headers (adj0[0], adj1[0],
1863                                     ip0, ip1, sizeof (ethernet_header_t));
1864
1865           if (is_midchain)
1866             {
1867               if (adj0->sub_type.midchain.fixup_func)
1868                 adj0->sub_type.midchain.fixup_func
1869                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1870               if (adj1->sub_type.midchain.fixup_func)
1871                 adj1->sub_type.midchain.fixup_func
1872                   (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1873             }
1874           if (is_mcast)
1875             {
1876               /*
1877                * copy bytes from the IP address into the MAC rewrite
1878                */
1879               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1880                                           adj0->
1881                                           rewrite_header.dst_mcast_offset,
1882                                           &ip0->dst_address.as_u32[3],
1883                                           (u8 *) ip0);
1884               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1885                                           adj1->
1886                                           rewrite_header.dst_mcast_offset,
1887                                           &ip1->dst_address.as_u32[3],
1888                                           (u8 *) ip1);
1889             }
1890
1891           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1892                                            to_next, n_left_to_next,
1893                                            pi0, pi1, next0, next1);
1894         }
1895
1896       while (n_left_from > 0 && n_left_to_next > 0)
1897         {
1898           ip_adjacency_t *adj0;
1899           vlib_buffer_t *p0;
1900           ip6_header_t *ip0;
1901           u32 pi0, rw_len0;
1902           u32 adj_index0, next0, error0;
1903           u32 tx_sw_if_index0;
1904           bool is_locally_originated0;
1905
1906           pi0 = to_next[0] = from[0];
1907
1908           p0 = vlib_get_buffer (vm, pi0);
1909
1910           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1911
1912           adj0 = adj_get (adj_index0);
1913
1914           ip0 = vlib_buffer_get_current (p0);
1915
1916           error0 = IP6_ERROR_NONE;
1917           next0 = IP6_REWRITE_NEXT_DROP;
1918
1919           /* Check hop limit */
1920           is_locally_originated0 =
1921             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1922           if (PREDICT_TRUE (!is_locally_originated0))
1923             {
1924               i32 hop_limit0 = ip0->hop_limit;
1925
1926               ASSERT (ip0->hop_limit > 0);
1927
1928               hop_limit0 -= 1;
1929
1930               ip0->hop_limit = hop_limit0;
1931
1932               if (PREDICT_FALSE (hop_limit0 <= 0))
1933                 {
1934                   /*
1935                    * If the hop count drops below 1 when forwarding, generate
1936                    * an ICMP response.
1937                    */
1938                   error0 = IP6_ERROR_TIME_EXPIRED;
1939                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1940                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1941                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1942                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1943                                                0);
1944                 }
1945             }
1946           else
1947             {
1948               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1949             }
1950
1951           if (is_midchain)
1952             {
1953               calc_checksums (vm, p0);
1954             }
1955
1956           /* Guess we are only writing on simple Ethernet header. */
1957           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
1958
1959           /* Update packet buffer attributes/set output interface. */
1960           rw_len0 = adj0[0].rewrite_header.data_bytes;
1961           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1962
1963           if (do_counters)
1964             {
1965               vlib_increment_combined_counter
1966                 (&adjacency_counters,
1967                  thread_index, adj_index0, 1,
1968                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1969             }
1970
1971           /* Check MTU of outgoing interface. */
1972           u16 ip0_len =
1973             clib_net_to_host_u16 (ip0->payload_length) +
1974             sizeof (ip6_header_t);
1975           if (p0->flags & VNET_BUFFER_F_GSO)
1976             ip0_len = gso_mtu_sz (p0);
1977
1978           ip6_mtu_check (p0, ip0_len,
1979                          adj0[0].rewrite_header.max_l3_packet_bytes,
1980                          is_locally_originated0, &next0, is_midchain,
1981                          &error0);
1982
1983           /* Don't adjust the buffer for hop count issue; icmp-error node
1984            * wants to see the IP header */
1985           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1986             {
1987               p0->current_data -= rw_len0;
1988               p0->current_length += rw_len0;
1989
1990               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1991
1992               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1993               next0 = adj0[0].rewrite_header.next_index;
1994
1995               if (PREDICT_FALSE
1996                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1997                 vnet_feature_arc_start (lm->output_feature_arc_index,
1998                                         tx_sw_if_index0, &next0, p0);
1999             }
2000           else
2001             {
2002               p0->error = error_node->errors[error0];
2003             }
2004
2005           if (is_midchain)
2006             {
2007               if (adj0->sub_type.midchain.fixup_func)
2008                 adj0->sub_type.midchain.fixup_func
2009                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2010             }
2011           if (is_mcast)
2012             {
2013               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
2014                                           adj0->
2015                                           rewrite_header.dst_mcast_offset,
2016                                           &ip0->dst_address.as_u32[3],
2017                                           (u8 *) ip0);
2018             }
2019
2020           from += 1;
2021           n_left_from -= 1;
2022           to_next += 1;
2023           n_left_to_next -= 1;
2024
2025           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2026                                            to_next, n_left_to_next,
2027                                            pi0, next0);
2028         }
2029
2030       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2031     }
2032
2033   /* Need to do trace after rewrites to pick up new packet data. */
2034   if (node->flags & VLIB_NODE_FLAG_TRACE)
2035     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2036
2037   return frame->n_vectors;
2038 }
2039
2040 always_inline uword
2041 ip6_rewrite_inline (vlib_main_t * vm,
2042                     vlib_node_runtime_t * node,
2043                     vlib_frame_t * frame,
2044                     int do_counters, int is_midchain, int is_mcast)
2045 {
2046   return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2047                                       is_midchain, is_mcast);
2048 }
2049
2050 VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm,
2051                                  vlib_node_runtime_t * node,
2052                                  vlib_frame_t * frame)
2053 {
2054   if (adj_are_counters_enabled ())
2055     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2056   else
2057     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2058 }
2059
2060 VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm,
2061                                        vlib_node_runtime_t * node,
2062                                        vlib_frame_t * frame)
2063 {
2064   if (adj_are_counters_enabled ())
2065     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2066   else
2067     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2068 }
2069
2070 VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm,
2071                                        vlib_node_runtime_t * node,
2072                                        vlib_frame_t * frame)
2073 {
2074   if (adj_are_counters_enabled ())
2075     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2076   else
2077     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2078 }
2079
2080 VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm,
2081                                   vlib_node_runtime_t * node,
2082                                   vlib_frame_t * frame)
2083 {
2084   if (adj_are_counters_enabled ())
2085     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2086   else
2087     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2088 }
2089
2090 VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
2091                                         vlib_node_runtime_t * node,
2092                                         vlib_frame_t * frame)
2093 {
2094   if (adj_are_counters_enabled ())
2095     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2096   else
2097     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2098 }
2099
2100 /* *INDENT-OFF* */
2101 VLIB_REGISTER_NODE (ip6_midchain_node) =
2102 {
2103   .name = "ip6-midchain",
2104   .vector_size = sizeof (u32),
2105   .format_trace = format_ip6_forward_next_trace,
2106   .sibling_of = "ip6-rewrite",
2107   };
2108
2109 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2110 {
2111   .name = "ip6-rewrite",
2112   .vector_size = sizeof (u32),
2113   .format_trace = format_ip6_rewrite_trace,
2114   .n_next_nodes = IP6_REWRITE_N_NEXT,
2115   .next_nodes =
2116   {
2117     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2118     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2119     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2120   },
2121 };
2122
2123 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2124   .name = "ip6-rewrite-bcast",
2125   .vector_size = sizeof (u32),
2126
2127   .format_trace = format_ip6_rewrite_trace,
2128   .sibling_of = "ip6-rewrite",
2129 };
2130
2131 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2132 {
2133   .name = "ip6-rewrite-mcast",
2134   .vector_size = sizeof (u32),
2135   .format_trace = format_ip6_rewrite_trace,
2136   .sibling_of = "ip6-rewrite",
2137 };
2138
2139
2140 VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
2141 {
2142   .name = "ip6-mcast-midchain",
2143   .vector_size = sizeof (u32),
2144   .format_trace = format_ip6_rewrite_trace,
2145   .sibling_of = "ip6-rewrite",
2146 };
2147
2148 /* *INDENT-ON* */
2149
2150 /*
2151  * Hop-by-Hop handling
2152  */
2153 #ifndef CLIB_MARCH_VARIANT
2154 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2155 #endif /* CLIB_MARCH_VARIANT */
2156
2157 #define foreach_ip6_hop_by_hop_error \
2158 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2159 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2160 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2161
2162 /* *INDENT-OFF* */
2163 typedef enum
2164 {
2165 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2166   foreach_ip6_hop_by_hop_error
2167 #undef _
2168   IP6_HOP_BY_HOP_N_ERROR,
2169 } ip6_hop_by_hop_error_t;
2170 /* *INDENT-ON* */
2171
2172 /*
2173  * Primary h-b-h handler trace support
2174  * We work pretty hard on the problem for obvious reasons
2175  */
2176 typedef struct
2177 {
2178   u32 next_index;
2179   u32 trace_len;
2180   u8 option_data[256];
2181 } ip6_hop_by_hop_trace_t;
2182
2183 extern vlib_node_registration_t ip6_hop_by_hop_node;
2184
2185 static char *ip6_hop_by_hop_error_strings[] = {
2186 #define _(sym,string) string,
2187   foreach_ip6_hop_by_hop_error
2188 #undef _
2189 };
2190
2191 #ifndef CLIB_MARCH_VARIANT
2192 u8 *
2193 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2194 {
2195   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2196   int total_len = va_arg (*args, int);
2197   ip6_hop_by_hop_option_t *opt0, *limit0;
2198   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2199   u8 type0;
2200
2201   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2202               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2203
2204   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2205   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2206
2207   while (opt0 < limit0)
2208     {
2209       type0 = opt0->type;
2210       switch (type0)
2211         {
2212         case 0:         /* Pad, just stop */
2213           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2214           break;
2215
2216         default:
2217           if (hm->trace[type0])
2218             {
2219               s = (*hm->trace[type0]) (s, opt0);
2220             }
2221           else
2222             {
2223               s =
2224                 format (s, "\n    unrecognized option %d length %d", type0,
2225                         opt0->length);
2226             }
2227           opt0 =
2228             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2229                                          sizeof (ip6_hop_by_hop_option_t));
2230           break;
2231         }
2232     }
2233   return s;
2234 }
2235 #endif
2236
2237 static u8 *
2238 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2239 {
2240   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2241   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2242   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2243   ip6_hop_by_hop_header_t *hbh0;
2244   ip6_hop_by_hop_option_t *opt0, *limit0;
2245   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2246
2247   u8 type0;
2248
2249   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2250
2251   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2252               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2253
2254   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2255   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2256
2257   while (opt0 < limit0)
2258     {
2259       type0 = opt0->type;
2260       switch (type0)
2261         {
2262         case 0:         /* Pad, just stop */
2263           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2264           break;
2265
2266         default:
2267           if (hm->trace[type0])
2268             {
2269               s = (*hm->trace[type0]) (s, opt0);
2270             }
2271           else
2272             {
2273               s =
2274                 format (s, "\n    unrecognized option %d length %d", type0,
2275                         opt0->length);
2276             }
2277           opt0 =
2278             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2279                                          sizeof (ip6_hop_by_hop_option_t));
2280           break;
2281         }
2282     }
2283   return s;
2284 }
2285
2286 always_inline u8
2287 ip6_scan_hbh_options (vlib_buffer_t * b0,
2288                       ip6_header_t * ip0,
2289                       ip6_hop_by_hop_header_t * hbh0,
2290                       ip6_hop_by_hop_option_t * opt0,
2291                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2292 {
2293   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2294   u8 type0;
2295   u8 error0 = 0;
2296
2297   while (opt0 < limit0)
2298     {
2299       type0 = opt0->type;
2300       switch (type0)
2301         {
2302         case 0:         /* Pad1 */
2303           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2304           continue;
2305         case 1:         /* PadN */
2306           break;
2307         default:
2308           if (hm->options[type0])
2309             {
2310               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2311                 {
2312                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2313                   return (error0);
2314                 }
2315             }
2316           else
2317             {
2318               /* Unrecognized mandatory option, check the two high order bits */
2319               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2320                 {
2321                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2322                   break;
2323                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2324                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2325                   *next0 = IP_LOOKUP_NEXT_DROP;
2326                   break;
2327                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2328                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2329                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2330                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2331                                                ICMP6_parameter_problem_unrecognized_option,
2332                                                (u8 *) opt0 - (u8 *) ip0);
2333                   break;
2334                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2335                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2336                   if (!ip6_address_is_multicast (&ip0->dst_address))
2337                     {
2338                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2339                       icmp6_error_set_vnet_buffer (b0,
2340                                                    ICMP6_parameter_problem,
2341                                                    ICMP6_parameter_problem_unrecognized_option,
2342                                                    (u8 *) opt0 - (u8 *) ip0);
2343                     }
2344                   else
2345                     {
2346                       *next0 = IP_LOOKUP_NEXT_DROP;
2347                     }
2348                   break;
2349                 }
2350               return (error0);
2351             }
2352         }
2353       opt0 =
2354         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2355                                      sizeof (ip6_hop_by_hop_option_t));
2356     }
2357   return (error0);
2358 }
2359
2360 /*
2361  * Process the Hop-by-Hop Options header
2362  */
2363 VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
2364                                     vlib_node_runtime_t * node,
2365                                     vlib_frame_t * frame)
2366 {
2367   vlib_node_runtime_t *error_node =
2368     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2369   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2370   u32 n_left_from, *from, *to_next;
2371   ip_lookup_next_t next_index;
2372
2373   from = vlib_frame_vector_args (frame);
2374   n_left_from = frame->n_vectors;
2375   next_index = node->cached_next_index;
2376
2377   while (n_left_from > 0)
2378     {
2379       u32 n_left_to_next;
2380
2381       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2382
2383       while (n_left_from >= 4 && n_left_to_next >= 2)
2384         {
2385           u32 bi0, bi1;
2386           vlib_buffer_t *b0, *b1;
2387           u32 next0, next1;
2388           ip6_header_t *ip0, *ip1;
2389           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2390           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2391           u8 error0 = 0, error1 = 0;
2392
2393           /* Prefetch next iteration. */
2394           {
2395             vlib_buffer_t *p2, *p3;
2396
2397             p2 = vlib_get_buffer (vm, from[2]);
2398             p3 = vlib_get_buffer (vm, from[3]);
2399
2400             vlib_prefetch_buffer_header (p2, LOAD);
2401             vlib_prefetch_buffer_header (p3, LOAD);
2402
2403             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2404             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2405           }
2406
2407           /* Speculatively enqueue b0, b1 to the current next frame */
2408           to_next[0] = bi0 = from[0];
2409           to_next[1] = bi1 = from[1];
2410           from += 2;
2411           to_next += 2;
2412           n_left_from -= 2;
2413           n_left_to_next -= 2;
2414
2415           b0 = vlib_get_buffer (vm, bi0);
2416           b1 = vlib_get_buffer (vm, bi1);
2417
2418           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2419           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2420           ip_adjacency_t *adj0 = adj_get (adj_index0);
2421           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2422           ip_adjacency_t *adj1 = adj_get (adj_index1);
2423
2424           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2425           next0 = adj0->lookup_next_index;
2426           next1 = adj1->lookup_next_index;
2427
2428           ip0 = vlib_buffer_get_current (b0);
2429           ip1 = vlib_buffer_get_current (b1);
2430           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2431           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2432           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2433           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2434           limit0 =
2435             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2436                                          ((hbh0->length + 1) << 3));
2437           limit1 =
2438             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2439                                          ((hbh1->length + 1) << 3));
2440
2441           /*
2442            * Basic validity checks
2443            */
2444           if ((hbh0->length + 1) << 3 >
2445               clib_net_to_host_u16 (ip0->payload_length))
2446             {
2447               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2448               next0 = IP_LOOKUP_NEXT_DROP;
2449               goto outdual;
2450             }
2451           /* Scan the set of h-b-h options, process ones that we understand */
2452           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2453
2454           if ((hbh1->length + 1) << 3 >
2455               clib_net_to_host_u16 (ip1->payload_length))
2456             {
2457               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2458               next1 = IP_LOOKUP_NEXT_DROP;
2459               goto outdual;
2460             }
2461           /* Scan the set of h-b-h options, process ones that we understand */
2462           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2463
2464         outdual:
2465           /* Has the classifier flagged this buffer for special treatment? */
2466           if (PREDICT_FALSE
2467               ((error0 == 0)
2468                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2469             next0 = hm->next_override;
2470
2471           /* Has the classifier flagged this buffer for special treatment? */
2472           if (PREDICT_FALSE
2473               ((error1 == 0)
2474                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2475             next1 = hm->next_override;
2476
2477           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2478             {
2479               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2480                 {
2481                   ip6_hop_by_hop_trace_t *t =
2482                     vlib_add_trace (vm, node, b0, sizeof (*t));
2483                   u32 trace_len = (hbh0->length + 1) << 3;
2484                   t->next_index = next0;
2485                   /* Capture the h-b-h option verbatim */
2486                   trace_len =
2487                     trace_len <
2488                     ARRAY_LEN (t->option_data) ? trace_len :
2489                     ARRAY_LEN (t->option_data);
2490                   t->trace_len = trace_len;
2491                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2492                 }
2493               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2494                 {
2495                   ip6_hop_by_hop_trace_t *t =
2496                     vlib_add_trace (vm, node, b1, sizeof (*t));
2497                   u32 trace_len = (hbh1->length + 1) << 3;
2498                   t->next_index = next1;
2499                   /* Capture the h-b-h option verbatim */
2500                   trace_len =
2501                     trace_len <
2502                     ARRAY_LEN (t->option_data) ? trace_len :
2503                     ARRAY_LEN (t->option_data);
2504                   t->trace_len = trace_len;
2505                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2506                 }
2507
2508             }
2509
2510           b0->error = error_node->errors[error0];
2511           b1->error = error_node->errors[error1];
2512
2513           /* verify speculative enqueue, maybe switch current next frame */
2514           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2515                                            n_left_to_next, bi0, bi1, next0,
2516                                            next1);
2517         }
2518
2519       while (n_left_from > 0 && n_left_to_next > 0)
2520         {
2521           u32 bi0;
2522           vlib_buffer_t *b0;
2523           u32 next0;
2524           ip6_header_t *ip0;
2525           ip6_hop_by_hop_header_t *hbh0;
2526           ip6_hop_by_hop_option_t *opt0, *limit0;
2527           u8 error0 = 0;
2528
2529           /* Speculatively enqueue b0 to the current next frame */
2530           bi0 = from[0];
2531           to_next[0] = bi0;
2532           from += 1;
2533           to_next += 1;
2534           n_left_from -= 1;
2535           n_left_to_next -= 1;
2536
2537           b0 = vlib_get_buffer (vm, bi0);
2538           /*
2539            * Default use the next_index from the adjacency.
2540            * A HBH option rarely redirects to a different node
2541            */
2542           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2543           ip_adjacency_t *adj0 = adj_get (adj_index0);
2544           next0 = adj0->lookup_next_index;
2545
2546           ip0 = vlib_buffer_get_current (b0);
2547           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2548           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2549           limit0 =
2550             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2551                                          ((hbh0->length + 1) << 3));
2552
2553           /*
2554            * Basic validity checks
2555            */
2556           if ((hbh0->length + 1) << 3 >
2557               clib_net_to_host_u16 (ip0->payload_length))
2558             {
2559               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2560               next0 = IP_LOOKUP_NEXT_DROP;
2561               goto out0;
2562             }
2563
2564           /* Scan the set of h-b-h options, process ones that we understand */
2565           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2566
2567         out0:
2568           /* Has the classifier flagged this buffer for special treatment? */
2569           if (PREDICT_FALSE
2570               ((error0 == 0)
2571                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2572             next0 = hm->next_override;
2573
2574           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2575             {
2576               ip6_hop_by_hop_trace_t *t =
2577                 vlib_add_trace (vm, node, b0, sizeof (*t));
2578               u32 trace_len = (hbh0->length + 1) << 3;
2579               t->next_index = next0;
2580               /* Capture the h-b-h option verbatim */
2581               trace_len =
2582                 trace_len <
2583                 ARRAY_LEN (t->option_data) ? trace_len :
2584                 ARRAY_LEN (t->option_data);
2585               t->trace_len = trace_len;
2586               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2587             }
2588
2589           b0->error = error_node->errors[error0];
2590
2591           /* verify speculative enqueue, maybe switch current next frame */
2592           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2593                                            n_left_to_next, bi0, next0);
2594         }
2595       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2596     }
2597   return frame->n_vectors;
2598 }
2599
2600 /* *INDENT-OFF* */
2601 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2602 {
2603   .name = "ip6-hop-by-hop",
2604   .sibling_of = "ip6-lookup",
2605   .vector_size = sizeof (u32),
2606   .format_trace = format_ip6_hop_by_hop_trace,
2607   .type = VLIB_NODE_TYPE_INTERNAL,
2608   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2609   .error_strings = ip6_hop_by_hop_error_strings,
2610   .n_next_nodes = 0,
2611 };
2612 /* *INDENT-ON* */
2613
2614 static clib_error_t *
2615 ip6_hop_by_hop_init (vlib_main_t * vm)
2616 {
2617   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2618   clib_memset (hm->options, 0, sizeof (hm->options));
2619   clib_memset (hm->trace, 0, sizeof (hm->trace));
2620   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2621   return (0);
2622 }
2623
2624 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2625
2626 #ifndef CLIB_MARCH_VARIANT
2627 void
2628 ip6_hbh_set_next_override (uword next)
2629 {
2630   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2631
2632   hm->next_override = next;
2633 }
2634
2635 int
2636 ip6_hbh_register_option (u8 option,
2637                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2638                                       ip6_hop_by_hop_option_t * opt),
2639                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2640 {
2641   ip6_main_t *im = &ip6_main;
2642   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2643
2644   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2645
2646   /* Already registered */
2647   if (hm->options[option])
2648     return (-1);
2649
2650   hm->options[option] = options;
2651   hm->trace[option] = trace;
2652
2653   /* Set global variable */
2654   im->hbh_enabled = 1;
2655
2656   return (0);
2657 }
2658
2659 int
2660 ip6_hbh_unregister_option (u8 option)
2661 {
2662   ip6_main_t *im = &ip6_main;
2663   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2664
2665   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2666
2667   /* Not registered */
2668   if (!hm->options[option])
2669     return (-1);
2670
2671   hm->options[option] = NULL;
2672   hm->trace[option] = NULL;
2673
2674   /* Disable global knob if this was the last option configured */
2675   int i;
2676   bool found = false;
2677   for (i = 0; i < 256; i++)
2678     {
2679       if (hm->options[option])
2680         {
2681           found = true;
2682           break;
2683         }
2684     }
2685   if (!found)
2686     im->hbh_enabled = 0;
2687
2688   return (0);
2689 }
2690
2691 /* Global IP6 main. */
2692 ip6_main_t ip6_main;
2693 #endif
2694
2695 static clib_error_t *
2696 ip6_lookup_init (vlib_main_t * vm)
2697 {
2698   ip6_main_t *im = &ip6_main;
2699   clib_error_t *error;
2700   uword i;
2701
2702   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2703     return error;
2704
2705   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2706     {
2707       u32 j, i0, i1;
2708
2709       i0 = i / 32;
2710       i1 = i % 32;
2711
2712       for (j = 0; j < i0; j++)
2713         im->fib_masks[i].as_u32[j] = ~0;
2714
2715       if (i1)
2716         im->fib_masks[i].as_u32[i0] =
2717           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2718     }
2719
2720   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2721
2722   if (im->lookup_table_nbuckets == 0)
2723     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2724
2725   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2726
2727   if (im->lookup_table_size == 0)
2728     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2729
2730   clib_bihash_init_24_8 (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2731                          "ip6 FIB fwding table",
2732                          im->lookup_table_nbuckets, im->lookup_table_size);
2733   clib_bihash_init_24_8 (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2734                          "ip6 FIB non-fwding table",
2735                          im->lookup_table_nbuckets, im->lookup_table_size);
2736   clib_bihash_init_40_8 (&im->ip6_mtable.ip6_mhash,
2737                          "ip6 mFIB table",
2738                          im->lookup_table_nbuckets, im->lookup_table_size);
2739
2740   /* Create FIB with index 0 and table id of 0. */
2741   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2742                                      FIB_SOURCE_DEFAULT_ROUTE);
2743   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2744                                       MFIB_SOURCE_DEFAULT_ROUTE);
2745
2746   {
2747     pg_node_t *pn;
2748     pn = pg_get_node (ip6_lookup_node.index);
2749     pn->unformat_edit = unformat_pg_ip6_header;
2750   }
2751
2752   /* Unless explicitly configured, don't process HBH options */
2753   im->hbh_enabled = 0;
2754
2755   return error;
2756 }
2757
2758 VLIB_INIT_FUNCTION (ip6_lookup_init);
2759
2760 #ifndef CLIB_MARCH_VARIANT
2761 int
2762 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2763 {
2764   u32 fib_index;
2765
2766   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2767
2768   if (~0 == fib_index)
2769     return VNET_API_ERROR_NO_SUCH_FIB;
2770
2771   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2772                                   flow_hash_config);
2773
2774   return 0;
2775 }
2776 #endif
2777
2778 static clib_error_t *
2779 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2780                               unformat_input_t * input,
2781                               vlib_cli_command_t * cmd)
2782 {
2783   int matched = 0;
2784   u32 table_id = 0;
2785   u32 flow_hash_config = 0;
2786   int rv;
2787
2788   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2789     {
2790       if (unformat (input, "table %d", &table_id))
2791         matched = 1;
2792 #define _(a,v) \
2793     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2794       foreach_flow_hash_bit
2795 #undef _
2796         else
2797         break;
2798     }
2799
2800   if (matched == 0)
2801     return clib_error_return (0, "unknown input `%U'",
2802                               format_unformat_error, input);
2803
2804   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2805   switch (rv)
2806     {
2807     case 0:
2808       break;
2809
2810     case -1:
2811       return clib_error_return (0, "no such FIB table %d", table_id);
2812
2813     default:
2814       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2815       break;
2816     }
2817
2818   return 0;
2819 }
2820
2821 /*?
2822  * Configure the set of IPv6 fields used by the flow hash.
2823  *
2824  * @cliexpar
2825  * @parblock
2826  * Example of how to set the flow hash on a given table:
2827  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2828  *
2829  * Example of display the configured flow hash:
2830  * @cliexstart{show ip6 fib}
2831  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2832  * @::/0
2833  *   unicast-ip6-chain
2834  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2835  *     [0] [@0]: dpo-drop ip6
2836  * fe80::/10
2837  *   unicast-ip6-chain
2838  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2839  *     [0] [@2]: dpo-receive
2840  * ff02::1/128
2841  *   unicast-ip6-chain
2842  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2843  *     [0] [@2]: dpo-receive
2844  * ff02::2/128
2845  *   unicast-ip6-chain
2846  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2847  *     [0] [@2]: dpo-receive
2848  * ff02::16/128
2849  *   unicast-ip6-chain
2850  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2851  *     [0] [@2]: dpo-receive
2852  * ff02::1:ff00:0/104
2853  *   unicast-ip6-chain
2854  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2855  *     [0] [@2]: dpo-receive
2856  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2857  * @::/0
2858  *   unicast-ip6-chain
2859  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2860  *     [0] [@0]: dpo-drop ip6
2861  * @::a:1:1:0:4/126
2862  *   unicast-ip6-chain
2863  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2864  *     [0] [@4]: ipv6-glean: af_packet0
2865  * @::a:1:1:0:7/128
2866  *   unicast-ip6-chain
2867  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2868  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2869  * fe80::/10
2870  *   unicast-ip6-chain
2871  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2872  *     [0] [@2]: dpo-receive
2873  * fe80::fe:3eff:fe3e:9222/128
2874  *   unicast-ip6-chain
2875  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2876  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2877  * ff02::1/128
2878  *   unicast-ip6-chain
2879  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2880  *     [0] [@2]: dpo-receive
2881  * ff02::2/128
2882  *   unicast-ip6-chain
2883  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2884  *     [0] [@2]: dpo-receive
2885  * ff02::16/128
2886  *   unicast-ip6-chain
2887  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2888  *     [0] [@2]: dpo-receive
2889  * ff02::1:ff00:0/104
2890  *   unicast-ip6-chain
2891  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2892  *     [0] [@2]: dpo-receive
2893  * @cliexend
2894  * @endparblock
2895 ?*/
2896 /* *INDENT-OFF* */
2897 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
2898 {
2899   .path = "set ip6 flow-hash",
2900   .short_help =
2901   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2902   .function = set_ip6_flow_hash_command_fn,
2903 };
2904 /* *INDENT-ON* */
2905
2906 static clib_error_t *
2907 show_ip6_local_command_fn (vlib_main_t * vm,
2908                            unformat_input_t * input, vlib_cli_command_t * cmd)
2909 {
2910   ip6_main_t *im = &ip6_main;
2911   ip_lookup_main_t *lm = &im->lookup_main;
2912   int i;
2913
2914   vlib_cli_output (vm, "Protocols handled by ip6_local");
2915   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
2916     {
2917       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2918         {
2919
2920           u32 node_index = vlib_get_node (vm,
2921                                           ip6_local_node.index)->
2922             next_nodes[lm->local_next_by_ip_protocol[i]];
2923           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
2924                            node_index);
2925         }
2926     }
2927   return 0;
2928 }
2929
2930
2931
2932 /*?
2933  * Display the set of protocols handled by the local IPv6 stack.
2934  *
2935  * @cliexpar
2936  * Example of how to display local protocol table:
2937  * @cliexstart{show ip6 local}
2938  * Protocols handled by ip6_local
2939  * 17
2940  * 43
2941  * 58
2942  * 115
2943  * @cliexend
2944 ?*/
2945 /* *INDENT-OFF* */
2946 VLIB_CLI_COMMAND (show_ip6_local, static) =
2947 {
2948   .path = "show ip6 local",
2949   .function = show_ip6_local_command_fn,
2950   .short_help = "show ip6 local",
2951 };
2952 /* *INDENT-ON* */
2953
2954 #ifndef CLIB_MARCH_VARIANT
2955 int
2956 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2957                              u32 table_index)
2958 {
2959   vnet_main_t *vnm = vnet_get_main ();
2960   vnet_interface_main_t *im = &vnm->interface_main;
2961   ip6_main_t *ipm = &ip6_main;
2962   ip_lookup_main_t *lm = &ipm->lookup_main;
2963   vnet_classify_main_t *cm = &vnet_classify_main;
2964   ip6_address_t *if_addr;
2965
2966   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2967     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2968
2969   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2970     return VNET_API_ERROR_NO_SUCH_ENTRY;
2971
2972   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2973   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2974
2975   if_addr = ip6_interface_first_address (ipm, sw_if_index);
2976
2977   if (NULL != if_addr)
2978     {
2979       fib_prefix_t pfx = {
2980         .fp_len = 128,
2981         .fp_proto = FIB_PROTOCOL_IP6,
2982         .fp_addr.ip6 = *if_addr,
2983       };
2984       u32 fib_index;
2985
2986       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2987                                                        sw_if_index);
2988
2989
2990       if (table_index != (u32) ~ 0)
2991         {
2992           dpo_id_t dpo = DPO_INVALID;
2993
2994           dpo_set (&dpo,
2995                    DPO_CLASSIFY,
2996                    DPO_PROTO_IP6,
2997                    classify_dpo_create (DPO_PROTO_IP6, table_index));
2998
2999           fib_table_entry_special_dpo_add (fib_index,
3000                                            &pfx,
3001                                            FIB_SOURCE_CLASSIFY,
3002                                            FIB_ENTRY_FLAG_NONE, &dpo);
3003           dpo_reset (&dpo);
3004         }
3005       else
3006         {
3007           fib_table_entry_special_remove (fib_index,
3008                                           &pfx, FIB_SOURCE_CLASSIFY);
3009         }
3010     }
3011
3012   return 0;
3013 }
3014 #endif
3015
3016 static clib_error_t *
3017 set_ip6_classify_command_fn (vlib_main_t * vm,
3018                              unformat_input_t * input,
3019                              vlib_cli_command_t * cmd)
3020 {
3021   u32 table_index = ~0;
3022   int table_index_set = 0;
3023   u32 sw_if_index = ~0;
3024   int rv;
3025
3026   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3027     {
3028       if (unformat (input, "table-index %d", &table_index))
3029         table_index_set = 1;
3030       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3031                          vnet_get_main (), &sw_if_index))
3032         ;
3033       else
3034         break;
3035     }
3036
3037   if (table_index_set == 0)
3038     return clib_error_return (0, "classify table-index must be specified");
3039
3040   if (sw_if_index == ~0)
3041     return clib_error_return (0, "interface / subif must be specified");
3042
3043   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3044
3045   switch (rv)
3046     {
3047     case 0:
3048       break;
3049
3050     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3051       return clib_error_return (0, "No such interface");
3052
3053     case VNET_API_ERROR_NO_SUCH_ENTRY:
3054       return clib_error_return (0, "No such classifier table");
3055     }
3056   return 0;
3057 }
3058
3059 /*?
3060  * Assign a classification table to an interface. The classification
3061  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3062  * commands. Once the table is create, use this command to filter packets
3063  * on an interface.
3064  *
3065  * @cliexpar
3066  * Example of how to assign a classification table to an interface:
3067  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3068 ?*/
3069 /* *INDENT-OFF* */
3070 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3071 {
3072   .path = "set ip6 classify",
3073   .short_help =
3074   "set ip6 classify intfc <interface> table-index <classify-idx>",
3075   .function = set_ip6_classify_command_fn,
3076 };
3077 /* *INDENT-ON* */
3078
3079 static clib_error_t *
3080 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3081 {
3082   ip6_main_t *im = &ip6_main;
3083   uword heapsize = 0;
3084   u32 tmp;
3085   u32 nbuckets = 0;
3086
3087   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3088     {
3089       if (unformat (input, "hash-buckets %d", &tmp))
3090         nbuckets = tmp;
3091       else if (unformat (input, "heap-size %U",
3092                          unformat_memory_size, &heapsize))
3093         ;
3094       else
3095         return clib_error_return (0, "unknown input '%U'",
3096                                   format_unformat_error, input);
3097     }
3098
3099   im->lookup_table_nbuckets = nbuckets;
3100   im->lookup_table_size = heapsize;
3101
3102   return 0;
3103 }
3104
3105 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3106
3107 /*
3108  * fd.io coding-style-patch-verification: ON
3109  *
3110  * Local Variables:
3111  * eval: (c-set-style "gnu")
3112  * End:
3113  */