src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_link.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52 #include <vnet/classify/vnet_classify.h>
  53
  54 #ifndef CLIB_MARCH_VARIANT
  55 #include <vppinfra/bihash_template.c>
  56 #endif
  57 #include <vnet/ip/ip6_forward.h>
  58 #include <vnet/interface_output.h>
  59
  60 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  61 #define OI_DECAP   0x80000000
  62
  63 static void
  64 ip6_add_interface_prefix_routes (ip6_main_t * im,
  65                                  u32 sw_if_index,
  66                                  u32 fib_index,
  67                                  ip6_address_t * address, u32 address_length)
  68 {
  69   ip_lookup_main_t *lm = &im->lookup_main;
  70   ip_interface_prefix_t *if_prefix;
  71
  72   /* *INDENT-OFF* */
  73   ip_interface_prefix_key_t key = {
  74     .prefix = {
  75       .fp_len = address_length,
  76       .fp_proto = FIB_PROTOCOL_IP6,
  77       .fp_addr.ip6 = {
  78         .as_u64 = {
  79           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
  80           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
  81         },
  82       },
  83     },
  84     .sw_if_index = sw_if_index,
  85   };
  86   /* *INDENT-ON* */
  87
  88   /* If prefix already set on interface, just increment ref count & return */
  89   if_prefix = ip_get_interface_prefix (lm, &key);
  90   if (if_prefix)
  91     {
  92       if_prefix->ref_count += 1;
  93       return;
  94     }
  95
  96   /* New prefix - allocate a pool entry, initialize it, add to the hash */
  97   pool_get (lm->if_prefix_pool, if_prefix);
  98   if_prefix->ref_count = 1;
  99   clib_memcpy (&if_prefix->key, &key, sizeof (key));
 100   mhash_set (&lm->prefix_to_if_prefix_index, &key,
 101              if_prefix - lm->if_prefix_pool, 0 /* old value */ );
 102
 103   /* length < 128 - add glean */
 104   if (address_length < 128)
 105     {
 106       /* set the glean route for the prefix */
 107       fib_table_entry_update_one_path (fib_index, &key.prefix,
 108                                        FIB_SOURCE_INTERFACE,
 109                                        (FIB_ENTRY_FLAG_CONNECTED |
 110                                         FIB_ENTRY_FLAG_ATTACHED),
 111                                        DPO_PROTO_IP6,
 112                                        /* No next-hop address */
 113                                        NULL, sw_if_index,
 114                                        /* invalid FIB index */
 115                                        ~0, 1,
 116                                        /* no out-label stack */
 117                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 118     }
 119 }
 120
 121 static void
 122 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 123                           ip6_main_t * im, u32 fib_index,
 124                           ip_interface_address_t * a)
 125 {
 126   ip_lookup_main_t *lm = &im->lookup_main;
 127   ip6_address_t *address = ip_interface_address_get_address (lm, a);
 128   fib_prefix_t pfx = {
 129     .fp_len = a->address_length,
 130     .fp_proto = FIB_PROTOCOL_IP6,
 131     .fp_addr.ip6 = *address,
 132   };
 133
 134   /* set special routes for the prefix if needed */
 135   ip6_add_interface_prefix_routes (im, sw_if_index, fib_index,
 136                                    address, a->address_length);
 137
 138   pfx.fp_len = 128;
 139   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 140     {
 141       u32 classify_table_index =
 142         lm->classify_table_index_by_sw_if_index[sw_if_index];
 143       if (classify_table_index != (u32) ~ 0)
 144         {
 145           dpo_id_t dpo = DPO_INVALID;
 146
 147           dpo_set (&dpo,
 148                    DPO_CLASSIFY,
 149                    DPO_PROTO_IP6,
 150                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 151
 152           fib_table_entry_special_dpo_add (fib_index,
 153                                            &pfx,
 154                                            FIB_SOURCE_CLASSIFY,
 155                                            FIB_ENTRY_FLAG_NONE, &dpo);
 156           dpo_reset (&dpo);
 157         }
 158     }
 159
 160   fib_table_entry_update_one_path (fib_index, &pfx,
 161                                    FIB_SOURCE_INTERFACE,
 162                                    (FIB_ENTRY_FLAG_CONNECTED |
 163                                     FIB_ENTRY_FLAG_LOCAL),
 164                                    DPO_PROTO_IP6,
 165                                    &pfx.fp_addr,
 166                                    sw_if_index, ~0,
 167                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 168 }
 169
 170 static void
 171 ip6_del_interface_prefix_routes (ip6_main_t * im,
 172                                  u32 sw_if_index,
 173                                  u32 fib_index,
 174                                  ip6_address_t * address, u32 address_length)
 175 {
 176   ip_lookup_main_t *lm = &im->lookup_main;
 177   ip_interface_prefix_t *if_prefix;
 178
 179   /* *INDENT-OFF* */
 180   ip_interface_prefix_key_t key = {
 181     .prefix = {
 182       .fp_len = address_length,
 183       .fp_proto = FIB_PROTOCOL_IP6,
 184       .fp_addr.ip6 = {
 185         .as_u64 = {
 186           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
 187           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
 188         },
 189       },
 190     },
 191     .sw_if_index = sw_if_index,
 192   };
 193   /* *INDENT-ON* */
 194
 195   if_prefix = ip_get_interface_prefix (lm, &key);
 196   if (!if_prefix)
 197     {
 198       clib_warning ("Prefix not found while deleting %U",
 199                     format_ip4_address_and_length, address, address_length);
 200       return;
 201     }
 202
 203   /* If not deleting last intf addr in prefix, decrement ref count & return */
 204   if_prefix->ref_count -= 1;
 205   if (if_prefix->ref_count > 0)
 206     return;
 207
 208   /* length <= 128, delete glean route */
 209   if (address_length <= 128)
 210     {
 211       /* remove glean route for prefix */
 212       fib_table_entry_delete (fib_index, &key.prefix, FIB_SOURCE_INTERFACE);
 213     }
 214
 215   mhash_unset (&lm->prefix_to_if_prefix_index, &key, 0 /* old_value */ );
 216   pool_put (lm->if_prefix_pool, if_prefix);
 217 }
 218
 219 static void
 220 ip6_del_interface_routes (u32 sw_if_index, ip6_main_t * im,
 221                           u32 fib_index,
 222                           ip6_address_t * address, u32 address_length)
 223 {
 224   fib_prefix_t pfx = {
 225     .fp_len = 128,
 226     .fp_proto = FIB_PROTOCOL_IP6,
 227     .fp_addr.ip6 = *address,
 228   };
 229
 230   /* delete special routes for the prefix if needed */
 231   ip6_del_interface_prefix_routes (im, sw_if_index, fib_index,
 232                                    address, address_length);
 233
 234   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 235 }
 236
 237 #ifndef CLIB_MARCH_VARIANT
 238 void
 239 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 240 {
 241   ip6_main_t *im = &ip6_main;
 242
 243   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 244
 245   /*
 246    * enable/disable only on the 1<->0 transition
 247    */
 248   if (is_enable)
 249     {
 250       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 251         return;
 252     }
 253   else
 254     {
 255       /* The ref count is 0 when an address is removed from an interface that has
 256        * no address - this is not a ciritical error */
 257       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 258           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 259         return;
 260     }
 261
 262   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 263                                !is_enable, 0, 0);
 264
 265   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 266                                sw_if_index, !is_enable, 0, 0);
 267 }
 268
 269 /* get first interface address */
 270 ip6_address_t *
 271 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 272 {
 273   ip_lookup_main_t *lm = &im->lookup_main;
 274   ip_interface_address_t *ia = 0;
 275   ip6_address_t *result = 0;
 276
 277   /* *INDENT-OFF* */
 278   foreach_ip_interface_address (lm, ia, sw_if_index,
 279                                 1 /* honor unnumbered */,
 280   ({
 281     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 282     result = a;
 283     break;
 284   }));
 285   /* *INDENT-ON* */
 286   return result;
 287 }
 288
 289 clib_error_t *
 290 ip6_add_del_interface_address (vlib_main_t * vm,
 291                                u32 sw_if_index,
 292                                ip6_address_t * address,
 293                                u32 address_length, u32 is_del)
 294 {
 295   vnet_main_t *vnm = vnet_get_main ();
 296   ip6_main_t *im = &ip6_main;
 297   ip_lookup_main_t *lm = &im->lookup_main;
 298   clib_error_t *error;
 299   u32 if_address_index;
 300   ip6_address_fib_t ip6_af, *addr_fib = 0;
 301   const ip6_address_t *ll_addr;
 302
 303   /* local0 interface doesn't support IP addressing */
 304   if (sw_if_index == 0)
 305     {
 306       return
 307         clib_error_create ("local0 interface doesn't support IP addressing");
 308     }
 309
 310   if (ip6_address_is_link_local_unicast (address))
 311     {
 312       if (address_length != 128)
 313         {
 314           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 315           return
 316             clib_error_create
 317             ("prefix length of link-local address must be 128");
 318         }
 319       if (!is_del)
 320         {
 321           int rv;
 322
 323           rv = ip6_set_link_local_address (sw_if_index, address);
 324
 325           if (rv)
 326             {
 327               vnm->api_errno = rv;
 328               return clib_error_create ("address not assignable");
 329             }
 330         }
 331       else
 332         {
 333           ll_addr = ip6_get_link_local_address (sw_if_index);
 334           if (ip6_address_is_equal (ll_addr, address))
 335             {
 336               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 337               return clib_error_create ("address not deletable");
 338             }
 339           else
 340             {
 341               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 342               return clib_error_create ("address not found");
 343             }
 344         }
 345     }
 346
 347   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 348   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 349
 350   ip6_addr_fib_init (&ip6_af, address,
 351                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 352   vec_add1 (addr_fib, ip6_af);
 353
 354   /* *INDENT-OFF* */
 355   if (!is_del)
 356     {
 357       /* When adding an address check that it does not conflict
 358          with an existing address on any interface in this table. */
 359       ip_interface_address_t *ia;
 360       vnet_sw_interface_t *sif;
 361
 362       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 363       ({
 364           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 365               im->fib_index_by_sw_if_index[sif->sw_if_index])
 366             {
 367               foreach_ip_interface_address
 368                 (&im->lookup_main, ia, sif->sw_if_index,
 369                  0 /* honor unnumbered */ ,
 370                  ({
 371                    ip6_address_t * x =
 372                      ip_interface_address_get_address
 373                      (&im->lookup_main, ia);
 374                    if (ip6_destination_matches_route
 375                        (im, address, x, ia->address_length) ||
 376                        ip6_destination_matches_route (im,
 377                                                       x,
 378                                                       address,
 379                                                       address_length))
 380                      {
 381                        /* an intf may have >1 addr from the same prefix */
 382                        if ((sw_if_index == sif->sw_if_index) &&
 383                            (ia->address_length == address_length) &&
 384                            !ip6_address_is_equal (x, address))
 385                          continue;
 386
 387                        /* error if the length or intf was different */
 388                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 389                        return
 390                          clib_error_create
 391                          ("failed to add %U which conflicts with %U for interface %U",
 392                           format_ip6_address_and_length, address,
 393                           address_length,
 394                           format_ip6_address_and_length, x,
 395                           ia->address_length,
 396                           format_vnet_sw_if_index_name, vnm,
 397                           sif->sw_if_index);
 398                      }
 399                  }));
 400             }
 401       }));
 402     }
 403   /* *INDENT-ON* */
 404
 405   {
 406     uword elts_before = pool_elts (lm->if_address_pool);
 407
 408     error = ip_interface_address_add_del
 409       (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 410     if (error)
 411       goto done;
 412
 413     /* Pool did not grow: add duplicate address. */
 414     if (elts_before == pool_elts (lm->if_address_pool))
 415       goto done;
 416   }
 417
 418   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 419   if (!is_del)
 420     ip6_link_enable (sw_if_index);
 421
 422   /* intf addr routes are added/deleted on admin up/down */
 423   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index))
 424     {
 425       if (is_del)
 426         ip6_del_interface_routes (sw_if_index,
 427                                   im, ip6_af.fib_index, address,
 428                                   address_length);
 429       else
 430         ip6_add_interface_routes (vnm, sw_if_index,
 431                                   im, ip6_af.fib_index,
 432                                   pool_elt_at_index (lm->if_address_pool,
 433                                                      if_address_index));
 434     }
 435   {
 436     ip6_add_del_interface_address_callback_t *cb;
 437     vec_foreach (cb, im->add_del_interface_address_callbacks)
 438       cb->function (im, cb->function_opaque, sw_if_index,
 439                     address, address_length, if_address_index, is_del);
 440   }
 441   if (is_del)
 442     ip6_link_disable (sw_if_index);
 443
 444 done:
 445   vec_free (addr_fib);
 446   return error;
 447 }
 448
 449 #endif
 450
 451 static clib_error_t *
 452 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 453 {
 454   ip6_main_t *im = &ip6_main;
 455   ip_interface_address_t *ia;
 456   ip6_address_t *a;
 457   u32 is_admin_up, fib_index;
 458
 459   /* Fill in lookup tables with default table (0). */
 460   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 461
 462   vec_validate_init_empty (im->
 463                            lookup_main.if_address_pool_index_by_sw_if_index,
 464                            sw_if_index, ~0);
 465
 466   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 467
 468   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 469
 470   /* *INDENT-OFF* */
 471   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 472                                 0 /* honor unnumbered */,
 473   ({
 474     a = ip_interface_address_get_address (&im->lookup_main, ia);
 475     if (is_admin_up)
 476       ip6_add_interface_routes (vnm, sw_if_index,
 477                                 im, fib_index,
 478                                 ia);
 479     else
 480       ip6_del_interface_routes (sw_if_index, im, fib_index,
 481                                 a, ia->address_length);
 482   }));
 483   /* *INDENT-ON* */
 484
 485   return 0;
 486 }
 487
 488 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 489
 490 /* Built-in ip6 unicast rx feature path definition */
 491 /* *INDENT-OFF* */
 492 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 493 {
 494   .arc_name  = "ip6-unicast",
 495   .start_nodes = VNET_FEATURES ("ip6-input"),
 496   .last_in_arc = "ip6-lookup",
 497   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 498 };
 499
 500 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 501 {
 502   .arc_name = "ip6-unicast",
 503   .node_name = "ip6-flow-classify",
 504   .runs_before = VNET_FEATURES ("ip6-inacl"),
 505 };
 506
 507 VNET_FEATURE_INIT (ip6_inacl, static) =
 508 {
 509   .arc_name = "ip6-unicast",
 510   .node_name = "ip6-inacl",
 511   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 512 };
 513
 514 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 515 {
 516   .arc_name = "ip6-unicast",
 517   .node_name = "ip6-policer-classify",
 518   .runs_before = VNET_FEATURES ("ipsec6-input-feature"),
 519 };
 520
 521 VNET_FEATURE_INIT (ip6_ipsec, static) =
 522 {
 523   .arc_name = "ip6-unicast",
 524   .node_name = "ipsec6-input-feature",
 525   .runs_before = VNET_FEATURES ("l2tp-decap"),
 526 };
 527
 528 VNET_FEATURE_INIT (ip6_l2tp, static) =
 529 {
 530   .arc_name = "ip6-unicast",
 531   .node_name = "l2tp-decap",
 532   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 533 };
 534
 535 VNET_FEATURE_INIT (ip6_vpath, static) =
 536 {
 537   .arc_name = "ip6-unicast",
 538   .node_name = "vpath-input-ip6",
 539   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 540 };
 541
 542 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 543 {
 544   .arc_name = "ip6-unicast",
 545   .node_name = "ip6-vxlan-bypass",
 546   .runs_before = VNET_FEATURES ("ip6-lookup"),
 547 };
 548
 549 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 550 {
 551   .arc_name = "ip6-unicast",
 552   .node_name = "ip6-not-enabled",
 553   .runs_before = VNET_FEATURES ("ip6-lookup"),
 554 };
 555
 556 VNET_FEATURE_INIT (ip6_lookup, static) =
 557 {
 558   .arc_name = "ip6-unicast",
 559   .node_name = "ip6-lookup",
 560   .runs_before = 0,  /*last feature*/
 561 };
 562
 563 /* Built-in ip6 multicast rx feature path definition (none now) */
 564 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 565 {
 566   .arc_name  = "ip6-multicast",
 567   .start_nodes = VNET_FEATURES ("ip6-input"),
 568   .last_in_arc = "ip6-mfib-forward-lookup",
 569   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 570 };
 571
 572 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 573   .arc_name = "ip6-multicast",
 574   .node_name = "vpath-input-ip6",
 575   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 576 };
 577
 578 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 579   .arc_name = "ip6-multicast",
 580   .node_name = "ip6-not-enabled",
 581   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 582 };
 583
 584 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 585   .arc_name = "ip6-multicast",
 586   .node_name = "ip6-mfib-forward-lookup",
 587   .runs_before = 0, /* last feature */
 588 };
 589
 590 /* Built-in ip4 tx feature path definition */
 591 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 592 {
 593   .arc_name  = "ip6-output",
 594   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 595   .last_in_arc = "interface-output",
 596   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 597 };
 598
 599 VNET_FEATURE_INIT (ip6_outacl, static) = {
 600   .arc_name = "ip6-output",
 601   .node_name = "ip6-outacl",
 602   .runs_before = VNET_FEATURES ("ipsec6-output-feature"),
 603 };
 604
 605 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 606   .arc_name = "ip6-output",
 607   .node_name = "ipsec6-output-feature",
 608   .runs_before = VNET_FEATURES ("interface-output"),
 609 };
 610
 611 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 612   .arc_name = "ip6-output",
 613   .node_name = "interface-output",
 614   .runs_before = 0, /* not before any other features */
 615 };
 616 /* *INDENT-ON* */
 617
 618 static clib_error_t *
 619 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 620 {
 621   ip6_main_t *im = &ip6_main;
 622
 623   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 624   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 625
 626   if (!is_add)
 627     {
 628       /* Ensure that IPv6 is disabled */
 629       ip6_main_t *im6 = &ip6_main;
 630       ip_lookup_main_t *lm6 = &im6->lookup_main;
 631       ip_interface_address_t *ia = 0;
 632       ip6_address_t *address;
 633       vlib_main_t *vm = vlib_get_main ();
 634
 635       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 636       /* *INDENT-OFF* */
 637       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 638       ({
 639         address = ip_interface_address_get_address (lm6, ia);
 640         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 641       }));
 642       /* *INDENT-ON* */
 643       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 644     }
 645
 646   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 647                                is_add, 0, 0);
 648
 649   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 650                                sw_if_index, is_add, 0, 0);
 651
 652   return /* no error */ 0;
 653 }
 654
 655 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 656
 657 VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 658                                 vlib_node_runtime_t * node,
 659                                 vlib_frame_t * frame)
 660 {
 661   return ip6_lookup_inline (vm, node, frame);
 662 }
 663
 664 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 665
 666 /* *INDENT-OFF* */
 667 VLIB_REGISTER_NODE (ip6_lookup_node) =
 668 {
 669   .name = "ip6-lookup",
 670   .vector_size = sizeof (u32),
 671   .format_trace = format_ip6_lookup_trace,
 672   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 673   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 674 };
 675 /* *INDENT-ON* */
 676
 677 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
 678                                       vlib_node_runtime_t * node,
 679                                       vlib_frame_t * frame)
 680 {
 681   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 682   u32 n_left, *from;
 683   u32 thread_index = vm->thread_index;
 684   ip6_main_t *im = &ip6_main;
 685   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 686   u16 nexts[VLIB_FRAME_SIZE], *next;
 687
 688   from = vlib_frame_vector_args (frame);
 689   n_left = frame->n_vectors;
 690   next = nexts;
 691
 692   vlib_get_buffers (vm, from, bufs, n_left);
 693
 694   while (n_left >= 4)
 695     {
 696       const load_balance_t *lb0, *lb1;
 697       const ip6_header_t *ip0, *ip1;
 698       u32 lbi0, hc0, lbi1, hc1;
 699       const dpo_id_t *dpo0, *dpo1;
 700
 701       /* Prefetch next iteration. */
 702       {
 703         vlib_prefetch_buffer_header (b[2], STORE);
 704         vlib_prefetch_buffer_header (b[3], STORE);
 705
 706         CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), STORE);
 707         CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), STORE);
 708       }
 709
 710       ip0 = vlib_buffer_get_current (b[0]);
 711       ip1 = vlib_buffer_get_current (b[1]);
 712       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 713       lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
 714
 715       lb0 = load_balance_get (lbi0);
 716       lb1 = load_balance_get (lbi1);
 717
 718       /*
 719        * this node is for via FIBs we can re-use the hash value from the
 720        * to node if present.
 721        * We don't want to use the same hash value at each level in the recursion
 722        * graph as that would lead to polarisation
 723        */
 724       hc0 = hc1 = 0;
 725
 726       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 727         {
 728           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 729             {
 730               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 731                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 732             }
 733           else
 734             {
 735               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 736                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 737             }
 738           dpo0 = load_balance_get_fwd_bucket
 739             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 740         }
 741       else
 742         {
 743           dpo0 = load_balance_get_bucket_i (lb0, 0);
 744         }
 745       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 746         {
 747           if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash))
 748             {
 749               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 750                 vnet_buffer (b[1])->ip.flow_hash >> 1;
 751             }
 752           else
 753             {
 754               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 755                 ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 756             }
 757           dpo1 = load_balance_get_fwd_bucket
 758             (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 759         }
 760       else
 761         {
 762           dpo1 = load_balance_get_bucket_i (lb1, 0);
 763         }
 764
 765       next[0] = dpo0->dpoi_next_node;
 766       next[1] = dpo1->dpoi_next_node;
 767
 768       /* Only process the HBH Option Header if explicitly configured to do so */
 769       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 770         {
 771           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 772             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 773         }
 774       /* Only process the HBH Option Header if explicitly configured to do so */
 775       if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 776         {
 777           next[1] = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 778             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[1];
 779         }
 780
 781       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 782       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 783
 784       vlib_increment_combined_counter
 785         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 786       vlib_increment_combined_counter
 787         (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1]));
 788
 789       b += 2;
 790       next += 2;
 791       n_left -= 2;
 792     }
 793
 794   while (n_left > 0)
 795     {
 796       const load_balance_t *lb0;
 797       const ip6_header_t *ip0;
 798       const dpo_id_t *dpo0;
 799       u32 lbi0, hc0;
 800
 801       ip0 = vlib_buffer_get_current (b[0]);
 802       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 803
 804       lb0 = load_balance_get (lbi0);
 805
 806       hc0 = 0;
 807       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 808         {
 809           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 810             {
 811               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 812                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 813             }
 814           else
 815             {
 816               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 817                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 818             }
 819           dpo0 = load_balance_get_fwd_bucket
 820             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 821         }
 822       else
 823         {
 824           dpo0 = load_balance_get_bucket_i (lb0, 0);
 825         }
 826
 827       next[0] = dpo0->dpoi_next_node;
 828       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 829
 830       /* Only process the HBH Option Header if explicitly configured to do so */
 831       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 832         {
 833           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 834             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 835         }
 836
 837       vlib_increment_combined_counter
 838         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 839
 840       b += 1;
 841       next += 1;
 842       n_left -= 1;
 843     }
 844
 845   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
 846
 847   if (node->flags & VLIB_NODE_FLAG_TRACE)
 848     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 849
 850   return frame->n_vectors;
 851 }
 852
 853 /* *INDENT-OFF* */
 854 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 855 {
 856   .name = "ip6-load-balance",
 857   .vector_size = sizeof (u32),
 858   .sibling_of = "ip6-lookup",
 859   .format_trace = format_ip6_lookup_trace,
 860 };
 861 /* *INDENT-ON* */
 862
 863 typedef struct
 864 {
 865   /* Adjacency taken. */
 866   u32 adj_index;
 867   u32 flow_hash;
 868   u32 fib_index;
 869
 870   /* Packet data, possibly *after* rewrite. */
 871   u8 packet_data[128 - 1 * sizeof (u32)];
 872 }
 873 ip6_forward_next_trace_t;
 874
 875 #ifndef CLIB_MARCH_VARIANT
 876 u8 *
 877 format_ip6_forward_next_trace (u8 * s, va_list * args)
 878 {
 879   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 880   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 881   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 882   u32 indent = format_get_indent (s);
 883
 884   s = format (s, "%U%U",
 885               format_white_space, indent,
 886               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 887   return s;
 888 }
 889 #endif
 890
 891 static u8 *
 892 format_ip6_lookup_trace (u8 * s, va_list * args)
 893 {
 894   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 895   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 896   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 897   u32 indent = format_get_indent (s);
 898
 899   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 900               t->fib_index, t->adj_index, t->flow_hash);
 901   s = format (s, "\n%U%U",
 902               format_white_space, indent,
 903               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 904   return s;
 905 }
 906
 907
 908 static u8 *
 909 format_ip6_rewrite_trace (u8 * s, va_list * args)
 910 {
 911   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 912   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 913   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 914   u32 indent = format_get_indent (s);
 915
 916   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 917               t->fib_index, t->adj_index, format_ip_adjacency,
 918               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 919   s = format (s, "\n%U%U",
 920               format_white_space, indent,
 921               format_ip_adjacency_packet_data,
 922               t->packet_data, sizeof (t->packet_data));
 923   return s;
 924 }
 925
 926 /* Common trace function for all ip6-forward next nodes. */
 927 #ifndef CLIB_MARCH_VARIANT
 928 void
 929 ip6_forward_next_trace (vlib_main_t * vm,
 930                         vlib_node_runtime_t * node,
 931                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 932 {
 933   u32 *from, n_left;
 934   ip6_main_t *im = &ip6_main;
 935
 936   n_left = frame->n_vectors;
 937   from = vlib_frame_vector_args (frame);
 938
 939   while (n_left >= 4)
 940     {
 941       u32 bi0, bi1;
 942       vlib_buffer_t *b0, *b1;
 943       ip6_forward_next_trace_t *t0, *t1;
 944
 945       /* Prefetch next iteration. */
 946       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 947       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 948
 949       bi0 = from[0];
 950       bi1 = from[1];
 951
 952       b0 = vlib_get_buffer (vm, bi0);
 953       b1 = vlib_get_buffer (vm, bi1);
 954
 955       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 956         {
 957           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 958           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 959           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 960           t0->fib_index =
 961             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 962              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 963             vec_elt (im->fib_index_by_sw_if_index,
 964                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 965
 966           clib_memcpy_fast (t0->packet_data,
 967                             vlib_buffer_get_current (b0),
 968                             sizeof (t0->packet_data));
 969         }
 970       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 971         {
 972           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 973           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 974           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 975           t1->fib_index =
 976             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
 977              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
 978             vec_elt (im->fib_index_by_sw_if_index,
 979                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
 980
 981           clib_memcpy_fast (t1->packet_data,
 982                             vlib_buffer_get_current (b1),
 983                             sizeof (t1->packet_data));
 984         }
 985       from += 2;
 986       n_left -= 2;
 987     }
 988
 989   while (n_left >= 1)
 990     {
 991       u32 bi0;
 992       vlib_buffer_t *b0;
 993       ip6_forward_next_trace_t *t0;
 994
 995       bi0 = from[0];
 996
 997       b0 = vlib_get_buffer (vm, bi0);
 998
 999       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1000         {
1001           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1002           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1003           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1004           t0->fib_index =
1005             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1006              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1007             vec_elt (im->fib_index_by_sw_if_index,
1008                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1009
1010           clib_memcpy_fast (t0->packet_data,
1011                             vlib_buffer_get_current (b0),
1012                             sizeof (t0->packet_data));
1013         }
1014       from += 1;
1015       n_left -= 1;
1016     }
1017 }
1018
1019 /* Compute TCP/UDP/ICMP6 checksum in software. */
1020 u16
1021 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1022                                    ip6_header_t * ip0, int *bogus_lengthp)
1023 {
1024   ip_csum_t sum0 = 0;
1025   u16 payload_length, payload_length_host_byte_order;
1026   u32 i;
1027   u32 headers_size = sizeof (ip0[0]);
1028   u8 *data_this_buffer;
1029   u8 next_hdr = ip0->protocol;
1030
1031   ASSERT (bogus_lengthp);
1032   *bogus_lengthp = 0;
1033
1034   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1035   data_this_buffer = (u8 *) (ip0 + 1);
1036   payload_length = ip0->payload_length;
1037
1038   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
1039    * or UDP-Ping packets */
1040   if (PREDICT_FALSE (next_hdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1041     {
1042       u32 skip_bytes;
1043       ip6_hop_by_hop_ext_t *ext_hdr =
1044         (ip6_hop_by_hop_ext_t *) data_this_buffer;
1045
1046       /* validate really icmp6 next */
1047       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1048               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
1049
1050       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
1051       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
1052
1053       payload_length_host_byte_order -= skip_bytes;
1054       headers_size += skip_bytes;
1055
1056       /* pseudo-header adjustments:
1057        *   exclude ext header bytes from payload length
1058        *   use payload IP proto rather than ext header IP proto
1059        */
1060       payload_length = clib_host_to_net_u16 (payload_length_host_byte_order);
1061       next_hdr = ext_hdr->next_hdr;
1062     }
1063
1064   /* Initialize checksum with ip pseudo-header. */
1065   sum0 = payload_length + clib_host_to_net_u16 (next_hdr);
1066
1067   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1068     {
1069       sum0 = ip_csum_with_carry
1070         (sum0, clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1071       sum0 = ip_csum_with_carry
1072         (sum0, clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1073     }
1074
1075   if (p0)
1076     return ip_calculate_l4_checksum (vm, p0, sum0,
1077                                      payload_length_host_byte_order,
1078                                      (u8 *) ip0, headers_size, NULL);
1079   else
1080     return ip_calculate_l4_checksum (vm, 0, sum0,
1081                                      payload_length_host_byte_order, NULL, 0,
1082                                      data_this_buffer);
1083 }
1084
1085 u32
1086 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1087 {
1088   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1089   udp_header_t *udp0;
1090   u16 sum16;
1091   int bogus_length;
1092
1093   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1094   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1095           || ip0->protocol == IP_PROTOCOL_ICMP6
1096           || ip0->protocol == IP_PROTOCOL_UDP
1097           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1098
1099   udp0 = (void *) (ip0 + 1);
1100   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1101     {
1102       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1103                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1104       return p0->flags;
1105     }
1106
1107   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1108
1109   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1110                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1111
1112   return p0->flags;
1113 }
1114 #endif
1115
1116 /**
1117  * @brief returns number of links on which src is reachable.
1118  */
1119 always_inline int
1120 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1121 {
1122   const load_balance_t *lb0;
1123   index_t lbi;
1124   u32 fib_index;
1125
1126   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1127                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1128   fib_index =
1129     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1130     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1131
1132   lbi = ip6_fib_table_fwding_lookup (fib_index, &i->src_address);
1133   lb0 = load_balance_get (lbi);
1134
1135   return (fib_urpf_check_size (lb0->lb_urpf));
1136 }
1137
1138 always_inline u8
1139 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1140                            u32 * udp_offset0)
1141 {
1142   u32 proto0;
1143   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1144   if (proto0 != IP_PROTOCOL_UDP)
1145     {
1146       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1147       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1148     }
1149   return proto0;
1150 }
1151
1152 /* *INDENT-OFF* */
1153 VNET_FEATURE_ARC_INIT (ip6_local) =
1154 {
1155   .arc_name  = "ip6-local",
1156   .start_nodes = VNET_FEATURES ("ip6-local"),
1157 };
1158 /* *INDENT-ON* */
1159
1160 static_always_inline u8
1161 ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
1162 {
1163
1164   u16 payload_length_host_byte_order;
1165   u32 n_this_buffer, n_bytes_left;
1166   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1167   u32 headers_size = sizeof (ip0[0]);
1168   u8 *data_this_buffer;
1169
1170
1171   data_this_buffer = (u8 *) (ip0 + 1);
1172
1173   ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t *) data_this_buffer;
1174
1175   /* validate really icmp6 next */
1176
1177   if (!(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1178       || (ext_hdr->next_hdr == IP_PROTOCOL_UDP))
1179     return 0;
1180
1181
1182   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1183   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1184
1185
1186   u32 n_ip_bytes_this_buffer =
1187     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1188   if (n_this_buffer + headers_size > n_ip_bytes_this_buffer)
1189     {
1190       n_this_buffer = p0->current_length > headers_size ?
1191         n_ip_bytes_this_buffer - headers_size : 0;
1192     }
1193
1194   n_bytes_left -= n_this_buffer;
1195   n_bytes_left -= p0->total_length_not_including_first_buffer;
1196
1197   if (n_bytes_left == 0)
1198     return 0;
1199   else
1200     return 1;
1201 }
1202
1203
1204 always_inline uword
1205 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1206                   vlib_frame_t * frame, int head_of_feature_arc)
1207 {
1208   ip6_main_t *im = &ip6_main;
1209   ip_lookup_main_t *lm = &im->lookup_main;
1210   u32 *from, n_left_from;
1211   vlib_node_runtime_t *error_node =
1212     vlib_node_get_runtime (vm, ip6_input_node.index);
1213   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1214   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1215   u16 nexts[VLIB_FRAME_SIZE], *next;
1216
1217   from = vlib_frame_vector_args (frame);
1218   n_left_from = frame->n_vectors;
1219
1220   if (node->flags & VLIB_NODE_FLAG_TRACE)
1221     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1222
1223   vlib_get_buffers (vm, from, bufs, n_left_from);
1224   b = bufs;
1225   next = nexts;
1226
1227   while (n_left_from > 2)
1228     {
1229       /* Prefetch next iteration. */
1230       if (n_left_from >= 6)
1231         {
1232           vlib_prefetch_buffer_header (b[4], STORE);
1233           vlib_prefetch_buffer_header (b[5], STORE);
1234           vlib_prefetch_buffer_data (b[2], LOAD);
1235           vlib_prefetch_buffer_data (b[3], LOAD);
1236         }
1237
1238       u8 error[2];
1239       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1240       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1241
1242       ip6_header_t *ip[2];
1243       ip[0] = vlib_buffer_get_current (b[0]);
1244       ip[1] = vlib_buffer_get_current (b[1]);
1245
1246       if (head_of_feature_arc)
1247         {
1248           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1249           vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1250
1251           u8 type[2];
1252           type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol];
1253           type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol];
1254
1255           u32 flags[2];
1256           flags[0] = b[0]->flags;
1257           flags[1] = b[1]->flags;
1258
1259           u32 good_l4_csum[2];
1260           good_l4_csum[0] =
1261             flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1262                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1263                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1264           good_l4_csum[1] =
1265             flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1266                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1267                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1268
1269           u32 udp_offset[2] = { };
1270           u8 is_tcp_udp[2];
1271           is_tcp_udp[0] =
1272             ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]);
1273           is_tcp_udp[1] =
1274             ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]);
1275           i16 len_diff[2] = { 0 };
1276           if (PREDICT_TRUE (is_tcp_udp[0]))
1277             {
1278               udp_header_t *udp =
1279                 (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]);
1280               good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP
1281                 && udp->checksum == 0;
1282               /* optimistically verify UDP length. */
1283               u16 ip_len, udp_len;
1284               ip_len = clib_net_to_host_u16 (ip[0]->payload_length);
1285               udp_len = clib_net_to_host_u16 (udp->length);
1286               len_diff[0] = ip_len - udp_len;
1287             }
1288           if (PREDICT_TRUE (is_tcp_udp[1]))
1289             {
1290               udp_header_t *udp =
1291                 (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]);
1292               good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP
1293                 && udp->checksum == 0;
1294               /* optimistically verify UDP length. */
1295               u16 ip_len, udp_len;
1296               ip_len = clib_net_to_host_u16 (ip[1]->payload_length);
1297               udp_len = clib_net_to_host_u16 (udp->length);
1298               len_diff[1] = ip_len - udp_len;
1299             }
1300
1301           good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1302           good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1303
1304           len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0;
1305           len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0;
1306
1307           u8 need_csum[2];
1308           need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN
1309             && !good_l4_csum[0]
1310             && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1311           need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN
1312             && !good_l4_csum[1]
1313             && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1314           if (PREDICT_FALSE (need_csum[0]))
1315             {
1316               flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1317               good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1318               error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1319             }
1320           else
1321             {
1322               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1323                 error[0] = IP6_ERROR_BAD_LENGTH;
1324             }
1325           if (PREDICT_FALSE (need_csum[1]))
1326             {
1327               flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]);
1328               good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1329               error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1330             }
1331           else
1332             {
1333               if (ip6_tcp_udp_icmp_bad_length (vm, b[1]))
1334                 error[1] = IP6_ERROR_BAD_LENGTH;
1335             }
1336
1337
1338           error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0];
1339
1340           error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1];
1341
1342           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1343                          IP6_ERROR_UDP_CHECKSUM,
1344                          "Wrong IP6 errors constants");
1345           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1346                          IP6_ERROR_ICMP_CHECKSUM,
1347                          "Wrong IP6 errors constants");
1348
1349           error[0] =
1350             !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0];
1351           error[1] =
1352             !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1];
1353
1354           /* Drop packets from unroutable hosts. */
1355           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1356           u8 unroutable[2];
1357           unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL
1358             && type[0] != IP_BUILTIN_PROTOCOL_ICMP
1359             && !ip6_address_is_link_local_unicast (&ip[0]->src_address);
1360           unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL
1361             && type[1] != IP_BUILTIN_PROTOCOL_ICMP
1362             && !ip6_address_is_link_local_unicast (&ip[1]->src_address);
1363           if (PREDICT_FALSE (unroutable[0]))
1364             {
1365               error[0] =
1366                 !ip6_urpf_loose_check (im, b[0],
1367                                        ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS
1368                 : error[0];
1369             }
1370           if (PREDICT_FALSE (unroutable[1]))
1371             {
1372               error[1] =
1373                 !ip6_urpf_loose_check (im, b[1],
1374                                        ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS
1375                 : error[1];
1376             }
1377
1378           vnet_buffer (b[0])->ip.fib_index =
1379             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1380             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1381             vnet_buffer (b[0])->ip.fib_index;
1382           vnet_buffer (b[1])->ip.fib_index =
1383             vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1384             vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1385             vnet_buffer (b[1])->ip.fib_index;
1386         }                       /* head_of_feature_arc */
1387
1388       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
1389       next[0] =
1390         error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1391       next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol];
1392       next[1] =
1393         error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1];
1394
1395       b[0]->error = error_node->errors[0];
1396       b[1]->error = error_node->errors[1];
1397
1398       if (head_of_feature_arc)
1399         {
1400           u8 ip6_unknown[2];
1401           ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1402           ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1403           if (PREDICT_TRUE (ip6_unknown[0]))
1404             {
1405               u32 next32 = next[0];
1406               vnet_feature_arc_start (arc_index,
1407                                       vnet_buffer (b[0])->sw_if_index
1408                                       [VLIB_RX], &next32, b[0]);
1409               next[0] = next32;
1410             }
1411           if (PREDICT_TRUE (ip6_unknown[1]))
1412             {
1413               u32 next32 = next[1];
1414               vnet_feature_arc_start (arc_index,
1415                                       vnet_buffer (b[1])->sw_if_index
1416                                       [VLIB_RX], &next32, b[1]);
1417               next[1] = next32;
1418             }
1419         }
1420
1421       /* next */
1422       b += 2;
1423       next += 2;
1424       n_left_from -= 2;
1425     }
1426
1427   while (n_left_from)
1428     {
1429       u8 error;
1430       error = IP6_ERROR_UNKNOWN_PROTOCOL;
1431
1432       ip6_header_t *ip;
1433       ip = vlib_buffer_get_current (b[0]);
1434
1435       if (head_of_feature_arc)
1436         {
1437           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1438           u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
1439
1440           u32 flags = b[0]->flags;
1441           u32 good_l4_csum =
1442             flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1443                      VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1444                      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1445
1446           u32 udp_offset;
1447           i16 len_diff = 0;
1448           u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
1449           if (PREDICT_TRUE (is_tcp_udp))
1450             {
1451               udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset);
1452               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1453               good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP
1454                 && udp->checksum == 0;
1455               /* optimistically verify UDP length. */
1456               u16 ip_len, udp_len;
1457               ip_len = clib_net_to_host_u16 (ip->payload_length);
1458               udp_len = clib_net_to_host_u16 (udp->length);
1459               len_diff = ip_len - udp_len;
1460             }
1461
1462           good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
1463           len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
1464
1465           u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
1466             && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1467           if (PREDICT_FALSE (need_csum))
1468             {
1469               flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1470               good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1471               error = IP6_ERROR_UNKNOWN_PROTOCOL;
1472             }
1473           else
1474             {
1475               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1476                 error = IP6_ERROR_BAD_LENGTH;
1477             }
1478
1479
1480
1481           error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
1482
1483           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1484                          IP6_ERROR_UDP_CHECKSUM,
1485                          "Wrong IP6 errors constants");
1486           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1487                          IP6_ERROR_ICMP_CHECKSUM,
1488                          "Wrong IP6 errors constants");
1489
1490           error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error;
1491
1492           /* Drop packets from unroutable hosts. */
1493           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1494           u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL
1495             && type != IP_BUILTIN_PROTOCOL_ICMP
1496             && !ip6_address_is_link_local_unicast (&ip->src_address);
1497           if (PREDICT_FALSE (unroutable))
1498             {
1499               error =
1500                 !ip6_urpf_loose_check (im, b[0],
1501                                        ip) ? IP6_ERROR_SRC_LOOKUP_MISS :
1502                 error;
1503             }
1504
1505           vnet_buffer (b[0])->ip.fib_index =
1506             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1507             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1508             vnet_buffer (b[0])->ip.fib_index;
1509         }                       /* head_of_feature_arc */
1510
1511       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
1512       next[0] =
1513         error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1514
1515       b[0]->error = error_node->errors[0];
1516
1517       if (head_of_feature_arc)
1518         {
1519           if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1520             {
1521               u32 next32 = next[0];
1522               vnet_feature_arc_start (arc_index,
1523                                       vnet_buffer (b[0])->sw_if_index
1524                                       [VLIB_RX], &next32, b[0]);
1525               next[0] = next32;
1526             }
1527         }
1528
1529       /* next */
1530       b += 1;
1531       next += 1;
1532       n_left_from -= 1;
1533     }
1534
1535   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1536   return frame->n_vectors;
1537 }
1538
1539 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1540                                vlib_frame_t * frame)
1541 {
1542   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1543 }
1544
1545 /* *INDENT-OFF* */
1546 VLIB_REGISTER_NODE (ip6_local_node) =
1547 {
1548   .name = "ip6-local",
1549   .vector_size = sizeof (u32),
1550   .format_trace = format_ip6_forward_next_trace,
1551   .n_next_nodes = IP_LOCAL_N_NEXT,
1552   .next_nodes =
1553   {
1554     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1555     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1556     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1557     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1558     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-full-reassembly",
1559   },
1560 };
1561 /* *INDENT-ON* */
1562
1563 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
1564                                           vlib_node_runtime_t * node,
1565                                           vlib_frame_t * frame)
1566 {
1567   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1568 }
1569
1570 /* *INDENT-OFF* */
1571 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
1572   .name = "ip6-local-end-of-arc",
1573   .vector_size = sizeof (u32),
1574
1575   .format_trace = format_ip6_forward_next_trace,
1576   .sibling_of = "ip6-local",
1577 };
1578
1579 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1580   .arc_name = "ip6-local",
1581   .node_name = "ip6-local-end-of-arc",
1582   .runs_before = 0, /* not before any other features */
1583 };
1584 /* *INDENT-ON* */
1585
1586 #ifdef CLIB_MARCH_VARIANT
1587 extern vlib_node_registration_t ip6_local_node;
1588 #else
1589 void
1590 ip6_register_protocol (u32 protocol, u32 node_index)
1591 {
1592   vlib_main_t *vm = vlib_get_main ();
1593   ip6_main_t *im = &ip6_main;
1594   ip_lookup_main_t *lm = &im->lookup_main;
1595
1596   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1597   lm->local_next_by_ip_protocol[protocol] =
1598     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1599 }
1600
1601 void
1602 ip6_unregister_protocol (u32 protocol)
1603 {
1604   ip6_main_t *im = &ip6_main;
1605   ip_lookup_main_t *lm = &im->lookup_main;
1606
1607   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1608   lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT;
1609 }
1610 #endif
1611
1612 typedef enum
1613 {
1614   IP6_REWRITE_NEXT_DROP,
1615   IP6_REWRITE_NEXT_ICMP_ERROR,
1616   IP6_REWRITE_NEXT_FRAGMENT,
1617   IP6_REWRITE_N_NEXT            /* Last */
1618 } ip6_rewrite_next_t;
1619
1620 /**
1621  * This bits of an IPv6 address to mask to construct a multicast
1622  * MAC address
1623  */
1624 #define IP6_MCAST_ADDR_MASK 0xffffffff
1625
1626 always_inline void
1627 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1628                u16 adj_packet_bytes, bool is_locally_generated,
1629                u32 * next, u8 is_midchain, u32 * error)
1630 {
1631   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1632     {
1633       if (is_locally_generated)
1634         {
1635           /* IP fragmentation */
1636           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1637                                    (is_midchain ?
1638                                     IP_FRAG_NEXT_IP_REWRITE_MIDCHAIN :
1639                                     IP_FRAG_NEXT_IP_REWRITE), 0);
1640           *next = IP6_REWRITE_NEXT_FRAGMENT;
1641           *error = IP6_ERROR_MTU_EXCEEDED;
1642         }
1643       else
1644         {
1645           *error = IP6_ERROR_MTU_EXCEEDED;
1646           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1647                                        adj_packet_bytes);
1648           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1649         }
1650     }
1651 }
1652
1653 always_inline uword
1654 ip6_rewrite_inline_with_gso (vlib_main_t * vm,
1655                              vlib_node_runtime_t * node,
1656                              vlib_frame_t * frame,
1657                              int do_counters, int is_midchain, int is_mcast)
1658 {
1659   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1660   u32 *from = vlib_frame_vector_args (frame);
1661   u32 n_left_from, n_left_to_next, *to_next, next_index;
1662   vlib_node_runtime_t *error_node =
1663     vlib_node_get_runtime (vm, ip6_input_node.index);
1664
1665   n_left_from = frame->n_vectors;
1666   next_index = node->cached_next_index;
1667   u32 thread_index = vm->thread_index;
1668
1669   while (n_left_from > 0)
1670     {
1671       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1672
1673       while (n_left_from >= 4 && n_left_to_next >= 2)
1674         {
1675           const ip_adjacency_t *adj0, *adj1;
1676           vlib_buffer_t *p0, *p1;
1677           ip6_header_t *ip0, *ip1;
1678           u32 pi0, rw_len0, next0, error0, adj_index0;
1679           u32 pi1, rw_len1, next1, error1, adj_index1;
1680           u32 tx_sw_if_index0, tx_sw_if_index1;
1681           bool is_locally_originated0, is_locally_originated1;
1682
1683           /* Prefetch next iteration. */
1684           {
1685             vlib_buffer_t *p2, *p3;
1686
1687             p2 = vlib_get_buffer (vm, from[2]);
1688             p3 = vlib_get_buffer (vm, from[3]);
1689
1690             vlib_prefetch_buffer_header (p2, LOAD);
1691             vlib_prefetch_buffer_header (p3, LOAD);
1692
1693             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1694             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1695
1696             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1697             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1698           }
1699
1700           pi0 = to_next[0] = from[0];
1701           pi1 = to_next[1] = from[1];
1702
1703           from += 2;
1704           n_left_from -= 2;
1705           to_next += 2;
1706           n_left_to_next -= 2;
1707
1708           p0 = vlib_get_buffer (vm, pi0);
1709           p1 = vlib_get_buffer (vm, pi1);
1710
1711           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1712           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1713
1714           ip0 = vlib_buffer_get_current (p0);
1715           ip1 = vlib_buffer_get_current (p1);
1716
1717           error0 = error1 = IP6_ERROR_NONE;
1718           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1719
1720           is_locally_originated0 =
1721             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1722           if (PREDICT_TRUE (!is_locally_originated0))
1723             {
1724               i32 hop_limit0 = ip0->hop_limit;
1725
1726               /* Input node should have reject packets with hop limit 0. */
1727               ASSERT (ip0->hop_limit > 0);
1728
1729               hop_limit0 -= 1;
1730
1731               ip0->hop_limit = hop_limit0;
1732
1733               /*
1734                * If the hop count drops below 1 when forwarding, generate
1735                * an ICMP response.
1736                */
1737               if (PREDICT_FALSE (hop_limit0 <= 0))
1738                 {
1739                   error0 = IP6_ERROR_TIME_EXPIRED;
1740                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1741                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1742                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1743                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1744                                                0);
1745                 }
1746             }
1747
1748           is_locally_originated1 =
1749             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1750           if (PREDICT_TRUE (!is_locally_originated1))
1751             {
1752               i32 hop_limit1 = ip1->hop_limit;
1753
1754               /* Input node should have reject packets with hop limit 0. */
1755               ASSERT (ip1->hop_limit > 0);
1756
1757               hop_limit1 -= 1;
1758
1759               ip1->hop_limit = hop_limit1;
1760
1761               /*
1762                * If the hop count drops below 1 when forwarding, generate
1763                * an ICMP response.
1764                */
1765               if (PREDICT_FALSE (hop_limit1 <= 0))
1766                 {
1767                   error1 = IP6_ERROR_TIME_EXPIRED;
1768                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1769                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1770                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1771                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1772                                                0);
1773                 }
1774             }
1775
1776           adj0 = adj_get (adj_index0);
1777           adj1 = adj_get (adj_index1);
1778
1779           rw_len0 = adj0[0].rewrite_header.data_bytes;
1780           rw_len1 = adj1[0].rewrite_header.data_bytes;
1781           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1782           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1783
1784           if (do_counters)
1785             {
1786               vlib_increment_combined_counter
1787                 (&adjacency_counters,
1788                  thread_index, adj_index0, 1,
1789                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1790               vlib_increment_combined_counter
1791                 (&adjacency_counters,
1792                  thread_index, adj_index1, 1,
1793                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1794             }
1795
1796           /* Check MTU of outgoing interface. */
1797           u16 ip0_len =
1798             clib_net_to_host_u16 (ip0->payload_length) +
1799             sizeof (ip6_header_t);
1800           u16 ip1_len =
1801             clib_net_to_host_u16 (ip1->payload_length) +
1802             sizeof (ip6_header_t);
1803           if (p0->flags & VNET_BUFFER_F_GSO)
1804             ip0_len = gso_mtu_sz (p0);
1805           if (p1->flags & VNET_BUFFER_F_GSO)
1806             ip1_len = gso_mtu_sz (p1);
1807
1808
1809
1810           ip6_mtu_check (p0, ip0_len,
1811                          adj0[0].rewrite_header.max_l3_packet_bytes,
1812                          is_locally_originated0, &next0, is_midchain,
1813                          &error0);
1814           ip6_mtu_check (p1, ip1_len,
1815                          adj1[0].rewrite_header.max_l3_packet_bytes,
1816                          is_locally_originated1, &next1, is_midchain,
1817                          &error1);
1818
1819           /* Don't adjust the buffer for hop count issue; icmp-error node
1820            * wants to see the IP header */
1821           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1822             {
1823               p0->current_data -= rw_len0;
1824               p0->current_length += rw_len0;
1825
1826               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1827               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1828               next0 = adj0[0].rewrite_header.next_index;
1829
1830               if (PREDICT_FALSE
1831                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1832                 vnet_feature_arc_start (lm->output_feature_arc_index,
1833                                         tx_sw_if_index0, &next0, p0);
1834             }
1835           else
1836             {
1837               p0->error = error_node->errors[error0];
1838             }
1839           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1840             {
1841               p1->current_data -= rw_len1;
1842               p1->current_length += rw_len1;
1843
1844               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1845               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1846               next1 = adj1[0].rewrite_header.next_index;
1847
1848               if (PREDICT_FALSE
1849                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1850                 vnet_feature_arc_start (lm->output_feature_arc_index,
1851                                         tx_sw_if_index1, &next1, p1);
1852             }
1853           else
1854             {
1855               p1->error = error_node->errors[error1];
1856             }
1857
1858           if (is_midchain)
1859             {
1860               /* before we paint on the next header, update the L4
1861                * checksums if required, since there's no offload on a tunnel */
1862               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
1863                                           1 /* is_ip6 */ ,
1864                                           0 /* with gso */ );
1865               vnet_calc_checksums_inline (vm, p1, 0 /* is_ip4 */ ,
1866                                           1 /* is_ip6 */ ,
1867                                           0 /* with gso */ );
1868             }
1869
1870           /* Guess we are only writing on simple Ethernet header. */
1871           vnet_rewrite_two_headers (adj0[0], adj1[0],
1872                                     ip0, ip1, sizeof (ethernet_header_t));
1873
1874           if (is_midchain)
1875             {
1876               if (adj0->sub_type.midchain.fixup_func)
1877                 adj0->sub_type.midchain.fixup_func
1878                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1879               if (adj1->sub_type.midchain.fixup_func)
1880                 adj1->sub_type.midchain.fixup_func
1881                   (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1882             }
1883           if (is_mcast)
1884             {
1885               /*
1886                * copy bytes from the IP address into the MAC rewrite
1887                */
1888               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1889                                           adj0->
1890                                           rewrite_header.dst_mcast_offset,
1891                                           &ip0->dst_address.as_u32[3],
1892                                           (u8 *) ip0);
1893               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1894                                           adj1->
1895                                           rewrite_header.dst_mcast_offset,
1896                                           &ip1->dst_address.as_u32[3],
1897                                           (u8 *) ip1);
1898             }
1899
1900           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1901                                            to_next, n_left_to_next,
1902                                            pi0, pi1, next0, next1);
1903         }
1904
1905       while (n_left_from > 0 && n_left_to_next > 0)
1906         {
1907           ip_adjacency_t *adj0;
1908           vlib_buffer_t *p0;
1909           ip6_header_t *ip0;
1910           u32 pi0, rw_len0;
1911           u32 adj_index0, next0, error0;
1912           u32 tx_sw_if_index0;
1913           bool is_locally_originated0;
1914
1915           pi0 = to_next[0] = from[0];
1916
1917           p0 = vlib_get_buffer (vm, pi0);
1918
1919           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1920
1921           adj0 = adj_get (adj_index0);
1922
1923           ip0 = vlib_buffer_get_current (p0);
1924
1925           error0 = IP6_ERROR_NONE;
1926           next0 = IP6_REWRITE_NEXT_DROP;
1927
1928           /* Check hop limit */
1929           is_locally_originated0 =
1930             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1931           if (PREDICT_TRUE (!is_locally_originated0))
1932             {
1933               i32 hop_limit0 = ip0->hop_limit;
1934
1935               ASSERT (ip0->hop_limit > 0);
1936
1937               hop_limit0 -= 1;
1938
1939               ip0->hop_limit = hop_limit0;
1940
1941               if (PREDICT_FALSE (hop_limit0 <= 0))
1942                 {
1943                   /*
1944                    * If the hop count drops below 1 when forwarding, generate
1945                    * an ICMP response.
1946                    */
1947                   error0 = IP6_ERROR_TIME_EXPIRED;
1948                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1949                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1950                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1951                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1952                                                0);
1953                 }
1954             }
1955
1956           if (is_midchain)
1957             {
1958               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
1959                                           1 /* is_ip6 */ ,
1960                                           0 /* with gso */ );
1961             }
1962
1963           /* Guess we are only writing on simple Ethernet header. */
1964           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
1965
1966           /* Update packet buffer attributes/set output interface. */
1967           rw_len0 = adj0[0].rewrite_header.data_bytes;
1968           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1969
1970           if (do_counters)
1971             {
1972               vlib_increment_combined_counter
1973                 (&adjacency_counters,
1974                  thread_index, adj_index0, 1,
1975                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1976             }
1977
1978           /* Check MTU of outgoing interface. */
1979           u16 ip0_len =
1980             clib_net_to_host_u16 (ip0->payload_length) +
1981             sizeof (ip6_header_t);
1982           if (p0->flags & VNET_BUFFER_F_GSO)
1983             ip0_len = gso_mtu_sz (p0);
1984
1985           ip6_mtu_check (p0, ip0_len,
1986                          adj0[0].rewrite_header.max_l3_packet_bytes,
1987                          is_locally_originated0, &next0, is_midchain,
1988                          &error0);
1989
1990           /* Don't adjust the buffer for hop count issue; icmp-error node
1991            * wants to see the IP header */
1992           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1993             {
1994               p0->current_data -= rw_len0;
1995               p0->current_length += rw_len0;
1996
1997               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1998
1999               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2000               next0 = adj0[0].rewrite_header.next_index;
2001
2002               if (PREDICT_FALSE
2003                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2004                 vnet_feature_arc_start (lm->output_feature_arc_index,
2005                                         tx_sw_if_index0, &next0, p0);
2006             }
2007           else
2008             {
2009               p0->error = error_node->errors[error0];
2010             }
2011
2012           if (is_midchain)
2013             {
2014               if (adj0->sub_type.midchain.fixup_func)
2015                 adj0->sub_type.midchain.fixup_func
2016                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2017             }
2018           if (is_mcast)
2019             {
2020               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
2021                                           adj0->
2022                                           rewrite_header.dst_mcast_offset,
2023                                           &ip0->dst_address.as_u32[3],
2024                                           (u8 *) ip0);
2025             }
2026
2027           from += 1;
2028           n_left_from -= 1;
2029           to_next += 1;
2030           n_left_to_next -= 1;
2031
2032           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2033                                            to_next, n_left_to_next,
2034                                            pi0, next0);
2035         }
2036
2037       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2038     }
2039
2040   /* Need to do trace after rewrites to pick up new packet data. */
2041   if (node->flags & VLIB_NODE_FLAG_TRACE)
2042     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2043
2044   return frame->n_vectors;
2045 }
2046
2047 always_inline uword
2048 ip6_rewrite_inline (vlib_main_t * vm,
2049                     vlib_node_runtime_t * node,
2050                     vlib_frame_t * frame,
2051                     int do_counters, int is_midchain, int is_mcast)
2052 {
2053   return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2054                                       is_midchain, is_mcast);
2055 }
2056
2057 VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm,
2058                                  vlib_node_runtime_t * node,
2059                                  vlib_frame_t * frame)
2060 {
2061   if (adj_are_counters_enabled ())
2062     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2063   else
2064     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2065 }
2066
2067 VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm,
2068                                        vlib_node_runtime_t * node,
2069                                        vlib_frame_t * frame)
2070 {
2071   if (adj_are_counters_enabled ())
2072     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2073   else
2074     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2075 }
2076
2077 VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm,
2078                                        vlib_node_runtime_t * node,
2079                                        vlib_frame_t * frame)
2080 {
2081   if (adj_are_counters_enabled ())
2082     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2083   else
2084     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2085 }
2086
2087 VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm,
2088                                   vlib_node_runtime_t * node,
2089                                   vlib_frame_t * frame)
2090 {
2091   if (adj_are_counters_enabled ())
2092     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2093   else
2094     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2095 }
2096
2097 VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
2098                                         vlib_node_runtime_t * node,
2099                                         vlib_frame_t * frame)
2100 {
2101   if (adj_are_counters_enabled ())
2102     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2103   else
2104     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2105 }
2106
2107 /* *INDENT-OFF* */
2108 VLIB_REGISTER_NODE (ip6_midchain_node) =
2109 {
2110   .name = "ip6-midchain",
2111   .vector_size = sizeof (u32),
2112   .format_trace = format_ip6_forward_next_trace,
2113   .sibling_of = "ip6-rewrite",
2114   };
2115
2116 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2117 {
2118   .name = "ip6-rewrite",
2119   .vector_size = sizeof (u32),
2120   .format_trace = format_ip6_rewrite_trace,
2121   .n_next_nodes = IP6_REWRITE_N_NEXT,
2122   .next_nodes =
2123   {
2124     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2125     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2126     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2127   },
2128 };
2129
2130 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2131   .name = "ip6-rewrite-bcast",
2132   .vector_size = sizeof (u32),
2133
2134   .format_trace = format_ip6_rewrite_trace,
2135   .sibling_of = "ip6-rewrite",
2136 };
2137
2138 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2139 {
2140   .name = "ip6-rewrite-mcast",
2141   .vector_size = sizeof (u32),
2142   .format_trace = format_ip6_rewrite_trace,
2143   .sibling_of = "ip6-rewrite",
2144 };
2145
2146
2147 VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
2148 {
2149   .name = "ip6-mcast-midchain",
2150   .vector_size = sizeof (u32),
2151   .format_trace = format_ip6_rewrite_trace,
2152   .sibling_of = "ip6-rewrite",
2153 };
2154
2155 /* *INDENT-ON* */
2156
2157 /*
2158  * Hop-by-Hop handling
2159  */
2160 #ifndef CLIB_MARCH_VARIANT
2161 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2162 #endif /* CLIB_MARCH_VARIANT */
2163
2164 #define foreach_ip6_hop_by_hop_error \
2165 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2166 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2167 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2168
2169 /* *INDENT-OFF* */
2170 typedef enum
2171 {
2172 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2173   foreach_ip6_hop_by_hop_error
2174 #undef _
2175   IP6_HOP_BY_HOP_N_ERROR,
2176 } ip6_hop_by_hop_error_t;
2177 /* *INDENT-ON* */
2178
2179 /*
2180  * Primary h-b-h handler trace support
2181  * We work pretty hard on the problem for obvious reasons
2182  */
2183 typedef struct
2184 {
2185   u32 next_index;
2186   u32 trace_len;
2187   u8 option_data[256];
2188 } ip6_hop_by_hop_trace_t;
2189
2190 extern vlib_node_registration_t ip6_hop_by_hop_node;
2191
2192 static char *ip6_hop_by_hop_error_strings[] = {
2193 #define _(sym,string) string,
2194   foreach_ip6_hop_by_hop_error
2195 #undef _
2196 };
2197
2198 #ifndef CLIB_MARCH_VARIANT
2199 u8 *
2200 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2201 {
2202   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2203   int total_len = va_arg (*args, int);
2204   ip6_hop_by_hop_option_t *opt0, *limit0;
2205   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2206   u8 type0;
2207
2208   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2209               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2210
2211   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2212   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2213
2214   while (opt0 < limit0)
2215     {
2216       type0 = opt0->type;
2217       switch (type0)
2218         {
2219         case 0:         /* Pad, just stop */
2220           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2221           break;
2222
2223         default:
2224           if (hm->trace[type0])
2225             {
2226               s = (*hm->trace[type0]) (s, opt0);
2227             }
2228           else
2229             {
2230               s =
2231                 format (s, "\n    unrecognized option %d length %d", type0,
2232                         opt0->length);
2233             }
2234           opt0 =
2235             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2236                                          sizeof (ip6_hop_by_hop_option_t));
2237           break;
2238         }
2239     }
2240   return s;
2241 }
2242 #endif
2243
2244 static u8 *
2245 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2246 {
2247   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2248   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2249   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2250   ip6_hop_by_hop_header_t *hbh0;
2251   ip6_hop_by_hop_option_t *opt0, *limit0;
2252   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2253
2254   u8 type0;
2255
2256   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2257
2258   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2259               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2260
2261   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2262   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2263
2264   while (opt0 < limit0)
2265     {
2266       type0 = opt0->type;
2267       switch (type0)
2268         {
2269         case 0:         /* Pad, just stop */
2270           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2271           break;
2272
2273         default:
2274           if (hm->trace[type0])
2275             {
2276               s = (*hm->trace[type0]) (s, opt0);
2277             }
2278           else
2279             {
2280               s =
2281                 format (s, "\n    unrecognized option %d length %d", type0,
2282                         opt0->length);
2283             }
2284           opt0 =
2285             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2286                                          sizeof (ip6_hop_by_hop_option_t));
2287           break;
2288         }
2289     }
2290   return s;
2291 }
2292
2293 always_inline u8
2294 ip6_scan_hbh_options (vlib_buffer_t * b0,
2295                       ip6_header_t * ip0,
2296                       ip6_hop_by_hop_header_t * hbh0,
2297                       ip6_hop_by_hop_option_t * opt0,
2298                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2299 {
2300   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2301   u8 type0;
2302   u8 error0 = 0;
2303
2304   while (opt0 < limit0)
2305     {
2306       type0 = opt0->type;
2307       switch (type0)
2308         {
2309         case 0:         /* Pad1 */
2310           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2311           continue;
2312         case 1:         /* PadN */
2313           break;
2314         default:
2315           if (hm->options[type0])
2316             {
2317               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2318                 {
2319                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2320                   return (error0);
2321                 }
2322             }
2323           else
2324             {
2325               /* Unrecognized mandatory option, check the two high order bits */
2326               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2327                 {
2328                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2329                   break;
2330                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2331                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2332                   *next0 = IP_LOOKUP_NEXT_DROP;
2333                   break;
2334                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2335                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2336                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2337                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2338                                                ICMP6_parameter_problem_unrecognized_option,
2339                                                (u8 *) opt0 - (u8 *) ip0);
2340                   break;
2341                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2342                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2343                   if (!ip6_address_is_multicast (&ip0->dst_address))
2344                     {
2345                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2346                       icmp6_error_set_vnet_buffer (b0,
2347                                                    ICMP6_parameter_problem,
2348                                                    ICMP6_parameter_problem_unrecognized_option,
2349                                                    (u8 *) opt0 - (u8 *) ip0);
2350                     }
2351                   else
2352                     {
2353                       *next0 = IP_LOOKUP_NEXT_DROP;
2354                     }
2355                   break;
2356                 }
2357               return (error0);
2358             }
2359         }
2360       opt0 =
2361         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2362                                      sizeof (ip6_hop_by_hop_option_t));
2363     }
2364   return (error0);
2365 }
2366
2367 /*
2368  * Process the Hop-by-Hop Options header
2369  */
2370 VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
2371                                     vlib_node_runtime_t * node,
2372                                     vlib_frame_t * frame)
2373 {
2374   vlib_node_runtime_t *error_node =
2375     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2376   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2377   u32 n_left_from, *from, *to_next;
2378   ip_lookup_next_t next_index;
2379
2380   from = vlib_frame_vector_args (frame);
2381   n_left_from = frame->n_vectors;
2382   next_index = node->cached_next_index;
2383
2384   while (n_left_from > 0)
2385     {
2386       u32 n_left_to_next;
2387
2388       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2389
2390       while (n_left_from >= 4 && n_left_to_next >= 2)
2391         {
2392           u32 bi0, bi1;
2393           vlib_buffer_t *b0, *b1;
2394           u32 next0, next1;
2395           ip6_header_t *ip0, *ip1;
2396           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2397           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2398           u8 error0 = 0, error1 = 0;
2399
2400           /* Prefetch next iteration. */
2401           {
2402             vlib_buffer_t *p2, *p3;
2403
2404             p2 = vlib_get_buffer (vm, from[2]);
2405             p3 = vlib_get_buffer (vm, from[3]);
2406
2407             vlib_prefetch_buffer_header (p2, LOAD);
2408             vlib_prefetch_buffer_header (p3, LOAD);
2409
2410             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2411             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2412           }
2413
2414           /* Speculatively enqueue b0, b1 to the current next frame */
2415           to_next[0] = bi0 = from[0];
2416           to_next[1] = bi1 = from[1];
2417           from += 2;
2418           to_next += 2;
2419           n_left_from -= 2;
2420           n_left_to_next -= 2;
2421
2422           b0 = vlib_get_buffer (vm, bi0);
2423           b1 = vlib_get_buffer (vm, bi1);
2424
2425           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2426           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2427           ip_adjacency_t *adj0 = adj_get (adj_index0);
2428           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2429           ip_adjacency_t *adj1 = adj_get (adj_index1);
2430
2431           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2432           next0 = adj0->lookup_next_index;
2433           next1 = adj1->lookup_next_index;
2434
2435           ip0 = vlib_buffer_get_current (b0);
2436           ip1 = vlib_buffer_get_current (b1);
2437           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2438           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2439           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2440           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2441           limit0 =
2442             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2443                                          ((hbh0->length + 1) << 3));
2444           limit1 =
2445             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2446                                          ((hbh1->length + 1) << 3));
2447
2448           /*
2449            * Basic validity checks
2450            */
2451           if ((hbh0->length + 1) << 3 >
2452               clib_net_to_host_u16 (ip0->payload_length))
2453             {
2454               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2455               next0 = IP_LOOKUP_NEXT_DROP;
2456               goto outdual;
2457             }
2458           /* Scan the set of h-b-h options, process ones that we understand */
2459           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2460
2461           if ((hbh1->length + 1) << 3 >
2462               clib_net_to_host_u16 (ip1->payload_length))
2463             {
2464               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2465               next1 = IP_LOOKUP_NEXT_DROP;
2466               goto outdual;
2467             }
2468           /* Scan the set of h-b-h options, process ones that we understand */
2469           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2470
2471         outdual:
2472           /* Has the classifier flagged this buffer for special treatment? */
2473           if (PREDICT_FALSE
2474               ((error0 == 0)
2475                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2476             next0 = hm->next_override;
2477
2478           /* Has the classifier flagged this buffer for special treatment? */
2479           if (PREDICT_FALSE
2480               ((error1 == 0)
2481                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2482             next1 = hm->next_override;
2483
2484           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2485             {
2486               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2487                 {
2488                   ip6_hop_by_hop_trace_t *t =
2489                     vlib_add_trace (vm, node, b0, sizeof (*t));
2490                   u32 trace_len = (hbh0->length + 1) << 3;
2491                   t->next_index = next0;
2492                   /* Capture the h-b-h option verbatim */
2493                   trace_len =
2494                     trace_len <
2495                     ARRAY_LEN (t->option_data) ? trace_len :
2496                     ARRAY_LEN (t->option_data);
2497                   t->trace_len = trace_len;
2498                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2499                 }
2500               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2501                 {
2502                   ip6_hop_by_hop_trace_t *t =
2503                     vlib_add_trace (vm, node, b1, sizeof (*t));
2504                   u32 trace_len = (hbh1->length + 1) << 3;
2505                   t->next_index = next1;
2506                   /* Capture the h-b-h option verbatim */
2507                   trace_len =
2508                     trace_len <
2509                     ARRAY_LEN (t->option_data) ? trace_len :
2510                     ARRAY_LEN (t->option_data);
2511                   t->trace_len = trace_len;
2512                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2513                 }
2514
2515             }
2516
2517           b0->error = error_node->errors[error0];
2518           b1->error = error_node->errors[error1];
2519
2520           /* verify speculative enqueue, maybe switch current next frame */
2521           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2522                                            n_left_to_next, bi0, bi1, next0,
2523                                            next1);
2524         }
2525
2526       while (n_left_from > 0 && n_left_to_next > 0)
2527         {
2528           u32 bi0;
2529           vlib_buffer_t *b0;
2530           u32 next0;
2531           ip6_header_t *ip0;
2532           ip6_hop_by_hop_header_t *hbh0;
2533           ip6_hop_by_hop_option_t *opt0, *limit0;
2534           u8 error0 = 0;
2535
2536           /* Speculatively enqueue b0 to the current next frame */
2537           bi0 = from[0];
2538           to_next[0] = bi0;
2539           from += 1;
2540           to_next += 1;
2541           n_left_from -= 1;
2542           n_left_to_next -= 1;
2543
2544           b0 = vlib_get_buffer (vm, bi0);
2545           /*
2546            * Default use the next_index from the adjacency.
2547            * A HBH option rarely redirects to a different node
2548            */
2549           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2550           ip_adjacency_t *adj0 = adj_get (adj_index0);
2551           next0 = adj0->lookup_next_index;
2552
2553           ip0 = vlib_buffer_get_current (b0);
2554           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2555           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2556           limit0 =
2557             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2558                                          ((hbh0->length + 1) << 3));
2559
2560           /*
2561            * Basic validity checks
2562            */
2563           if ((hbh0->length + 1) << 3 >
2564               clib_net_to_host_u16 (ip0->payload_length))
2565             {
2566               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2567               next0 = IP_LOOKUP_NEXT_DROP;
2568               goto out0;
2569             }
2570
2571           /* Scan the set of h-b-h options, process ones that we understand */
2572           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2573
2574         out0:
2575           /* Has the classifier flagged this buffer for special treatment? */
2576           if (PREDICT_FALSE
2577               ((error0 == 0)
2578                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2579             next0 = hm->next_override;
2580
2581           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2582             {
2583               ip6_hop_by_hop_trace_t *t =
2584                 vlib_add_trace (vm, node, b0, sizeof (*t));
2585               u32 trace_len = (hbh0->length + 1) << 3;
2586               t->next_index = next0;
2587               /* Capture the h-b-h option verbatim */
2588               trace_len =
2589                 trace_len <
2590                 ARRAY_LEN (t->option_data) ? trace_len :
2591                 ARRAY_LEN (t->option_data);
2592               t->trace_len = trace_len;
2593               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2594             }
2595
2596           b0->error = error_node->errors[error0];
2597
2598           /* verify speculative enqueue, maybe switch current next frame */
2599           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2600                                            n_left_to_next, bi0, next0);
2601         }
2602       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2603     }
2604   return frame->n_vectors;
2605 }
2606
2607 /* *INDENT-OFF* */
2608 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2609 {
2610   .name = "ip6-hop-by-hop",
2611   .sibling_of = "ip6-lookup",
2612   .vector_size = sizeof (u32),
2613   .format_trace = format_ip6_hop_by_hop_trace,
2614   .type = VLIB_NODE_TYPE_INTERNAL,
2615   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2616   .error_strings = ip6_hop_by_hop_error_strings,
2617   .n_next_nodes = 0,
2618 };
2619 /* *INDENT-ON* */
2620
2621 static clib_error_t *
2622 ip6_hop_by_hop_init (vlib_main_t * vm)
2623 {
2624   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2625   clib_memset (hm->options, 0, sizeof (hm->options));
2626   clib_memset (hm->trace, 0, sizeof (hm->trace));
2627   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2628   return (0);
2629 }
2630
2631 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2632
2633 #ifndef CLIB_MARCH_VARIANT
2634 void
2635 ip6_hbh_set_next_override (uword next)
2636 {
2637   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2638
2639   hm->next_override = next;
2640 }
2641
2642 int
2643 ip6_hbh_register_option (u8 option,
2644                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2645                                       ip6_hop_by_hop_option_t * opt),
2646                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2647 {
2648   ip6_main_t *im = &ip6_main;
2649   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2650
2651   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2652
2653   /* Already registered */
2654   if (hm->options[option])
2655     return (-1);
2656
2657   hm->options[option] = options;
2658   hm->trace[option] = trace;
2659
2660   /* Set global variable */
2661   im->hbh_enabled = 1;
2662
2663   return (0);
2664 }
2665
2666 int
2667 ip6_hbh_unregister_option (u8 option)
2668 {
2669   ip6_main_t *im = &ip6_main;
2670   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2671
2672   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2673
2674   /* Not registered */
2675   if (!hm->options[option])
2676     return (-1);
2677
2678   hm->options[option] = NULL;
2679   hm->trace[option] = NULL;
2680
2681   /* Disable global knob if this was the last option configured */
2682   int i;
2683   bool found = false;
2684   for (i = 0; i < 256; i++)
2685     {
2686       if (hm->options[option])
2687         {
2688           found = true;
2689           break;
2690         }
2691     }
2692   if (!found)
2693     im->hbh_enabled = 0;
2694
2695   return (0);
2696 }
2697
2698 /* Global IP6 main. */
2699 ip6_main_t ip6_main;
2700 #endif
2701
2702 static clib_error_t *
2703 ip6_lookup_init (vlib_main_t * vm)
2704 {
2705   ip6_main_t *im = &ip6_main;
2706   clib_error_t *error;
2707   uword i;
2708
2709   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2710     return error;
2711
2712   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2713     {
2714       u32 j, i0, i1;
2715
2716       i0 = i / 32;
2717       i1 = i % 32;
2718
2719       for (j = 0; j < i0; j++)
2720         im->fib_masks[i].as_u32[j] = ~0;
2721
2722       if (i1)
2723         im->fib_masks[i].as_u32[i0] =
2724           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2725     }
2726
2727   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2728
2729   if (im->lookup_table_nbuckets == 0)
2730     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2731
2732   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2733
2734   if (im->lookup_table_size == 0)
2735     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2736
2737   clib_bihash_init_24_8 (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2738                          "ip6 FIB fwding table",
2739                          im->lookup_table_nbuckets, im->lookup_table_size);
2740   clib_bihash_init_24_8 (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2741                          "ip6 FIB non-fwding table",
2742                          im->lookup_table_nbuckets, im->lookup_table_size);
2743   clib_bihash_init_40_8 (&im->ip6_mtable.ip6_mhash,
2744                          "ip6 mFIB table",
2745                          im->lookup_table_nbuckets, im->lookup_table_size);
2746
2747   /* Create FIB with index 0 and table id of 0. */
2748   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2749                                      FIB_SOURCE_DEFAULT_ROUTE);
2750   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2751                                       MFIB_SOURCE_DEFAULT_ROUTE);
2752
2753   {
2754     pg_node_t *pn;
2755     pn = pg_get_node (ip6_lookup_node.index);
2756     pn->unformat_edit = unformat_pg_ip6_header;
2757   }
2758
2759   /* Unless explicitly configured, don't process HBH options */
2760   im->hbh_enabled = 0;
2761
2762   return error;
2763 }
2764
2765 VLIB_INIT_FUNCTION (ip6_lookup_init);
2766
2767 #ifndef CLIB_MARCH_VARIANT
2768 int
2769 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2770 {
2771   u32 fib_index;
2772
2773   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2774
2775   if (~0 == fib_index)
2776     return VNET_API_ERROR_NO_SUCH_FIB;
2777
2778   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2779                                   flow_hash_config);
2780
2781   return 0;
2782 }
2783 #endif
2784
2785 static clib_error_t *
2786 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2787                               unformat_input_t * input,
2788                               vlib_cli_command_t * cmd)
2789 {
2790   int matched = 0;
2791   u32 table_id = 0;
2792   u32 flow_hash_config = 0;
2793   int rv;
2794
2795   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2796     {
2797       if (unformat (input, "table %d", &table_id))
2798         matched = 1;
2799 #define _(a,v) \
2800     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2801       foreach_flow_hash_bit
2802 #undef _
2803         else
2804         break;
2805     }
2806
2807   if (matched == 0)
2808     return clib_error_return (0, "unknown input `%U'",
2809                               format_unformat_error, input);
2810
2811   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2812   switch (rv)
2813     {
2814     case 0:
2815       break;
2816
2817     case -1:
2818       return clib_error_return (0, "no such FIB table %d", table_id);
2819
2820     default:
2821       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2822       break;
2823     }
2824
2825   return 0;
2826 }
2827
2828 /*?
2829  * Configure the set of IPv6 fields used by the flow hash.
2830  *
2831  * @cliexpar
2832  * @parblock
2833  * Example of how to set the flow hash on a given table:
2834  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2835  *
2836  * Example of display the configured flow hash:
2837  * @cliexstart{show ip6 fib}
2838  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2839  * @::/0
2840  *   unicast-ip6-chain
2841  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2842  *     [0] [@0]: dpo-drop ip6
2843  * fe80::/10
2844  *   unicast-ip6-chain
2845  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2846  *     [0] [@2]: dpo-receive
2847  * ff02::1/128
2848  *   unicast-ip6-chain
2849  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2850  *     [0] [@2]: dpo-receive
2851  * ff02::2/128
2852  *   unicast-ip6-chain
2853  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2854  *     [0] [@2]: dpo-receive
2855  * ff02::16/128
2856  *   unicast-ip6-chain
2857  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2858  *     [0] [@2]: dpo-receive
2859  * ff02::1:ff00:0/104
2860  *   unicast-ip6-chain
2861  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2862  *     [0] [@2]: dpo-receive
2863  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2864  * @::/0
2865  *   unicast-ip6-chain
2866  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2867  *     [0] [@0]: dpo-drop ip6
2868  * @::a:1:1:0:4/126
2869  *   unicast-ip6-chain
2870  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2871  *     [0] [@4]: ipv6-glean: af_packet0
2872  * @::a:1:1:0:7/128
2873  *   unicast-ip6-chain
2874  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2875  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2876  * fe80::/10
2877  *   unicast-ip6-chain
2878  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2879  *     [0] [@2]: dpo-receive
2880  * fe80::fe:3eff:fe3e:9222/128
2881  *   unicast-ip6-chain
2882  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2883  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2884  * ff02::1/128
2885  *   unicast-ip6-chain
2886  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2887  *     [0] [@2]: dpo-receive
2888  * ff02::2/128
2889  *   unicast-ip6-chain
2890  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2891  *     [0] [@2]: dpo-receive
2892  * ff02::16/128
2893  *   unicast-ip6-chain
2894  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2895  *     [0] [@2]: dpo-receive
2896  * ff02::1:ff00:0/104
2897  *   unicast-ip6-chain
2898  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2899  *     [0] [@2]: dpo-receive
2900  * @cliexend
2901  * @endparblock
2902 ?*/
2903 /* *INDENT-OFF* */
2904 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
2905 {
2906   .path = "set ip6 flow-hash",
2907   .short_help =
2908   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2909   .function = set_ip6_flow_hash_command_fn,
2910 };
2911 /* *INDENT-ON* */
2912
2913 static clib_error_t *
2914 show_ip6_local_command_fn (vlib_main_t * vm,
2915                            unformat_input_t * input, vlib_cli_command_t * cmd)
2916 {
2917   ip6_main_t *im = &ip6_main;
2918   ip_lookup_main_t *lm = &im->lookup_main;
2919   int i;
2920
2921   vlib_cli_output (vm, "Protocols handled by ip6_local");
2922   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
2923     {
2924       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2925         {
2926
2927           u32 node_index = vlib_get_node (vm,
2928                                           ip6_local_node.index)->
2929             next_nodes[lm->local_next_by_ip_protocol[i]];
2930           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
2931                            node_index);
2932         }
2933     }
2934   return 0;
2935 }
2936
2937
2938
2939 /*?
2940  * Display the set of protocols handled by the local IPv6 stack.
2941  *
2942  * @cliexpar
2943  * Example of how to display local protocol table:
2944  * @cliexstart{show ip6 local}
2945  * Protocols handled by ip6_local
2946  * 17
2947  * 43
2948  * 58
2949  * 115
2950  * @cliexend
2951 ?*/
2952 /* *INDENT-OFF* */
2953 VLIB_CLI_COMMAND (show_ip6_local, static) =
2954 {
2955   .path = "show ip6 local",
2956   .function = show_ip6_local_command_fn,
2957   .short_help = "show ip6 local",
2958 };
2959 /* *INDENT-ON* */
2960
2961 #ifndef CLIB_MARCH_VARIANT
2962 int
2963 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2964                              u32 table_index)
2965 {
2966   vnet_main_t *vnm = vnet_get_main ();
2967   vnet_interface_main_t *im = &vnm->interface_main;
2968   ip6_main_t *ipm = &ip6_main;
2969   ip_lookup_main_t *lm = &ipm->lookup_main;
2970   vnet_classify_main_t *cm = &vnet_classify_main;
2971   ip6_address_t *if_addr;
2972
2973   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2974     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2975
2976   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2977     return VNET_API_ERROR_NO_SUCH_ENTRY;
2978
2979   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2980   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
2981
2982   if_addr = ip6_interface_first_address (ipm, sw_if_index);
2983
2984   if (NULL != if_addr)
2985     {
2986       fib_prefix_t pfx = {
2987         .fp_len = 128,
2988         .fp_proto = FIB_PROTOCOL_IP6,
2989         .fp_addr.ip6 = *if_addr,
2990       };
2991       u32 fib_index;
2992
2993       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
2994                                                        sw_if_index);
2995
2996
2997       if (table_index != (u32) ~ 0)
2998         {
2999           dpo_id_t dpo = DPO_INVALID;
3000
3001           dpo_set (&dpo,
3002                    DPO_CLASSIFY,
3003                    DPO_PROTO_IP6,
3004                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3005
3006           fib_table_entry_special_dpo_add (fib_index,
3007                                            &pfx,
3008                                            FIB_SOURCE_CLASSIFY,
3009                                            FIB_ENTRY_FLAG_NONE, &dpo);
3010           dpo_reset (&dpo);
3011         }
3012       else
3013         {
3014           fib_table_entry_special_remove (fib_index,
3015                                           &pfx, FIB_SOURCE_CLASSIFY);
3016         }
3017     }
3018
3019   return 0;
3020 }
3021 #endif
3022
3023 static clib_error_t *
3024 set_ip6_classify_command_fn (vlib_main_t * vm,
3025                              unformat_input_t * input,
3026                              vlib_cli_command_t * cmd)
3027 {
3028   u32 table_index = ~0;
3029   int table_index_set = 0;
3030   u32 sw_if_index = ~0;
3031   int rv;
3032
3033   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3034     {
3035       if (unformat (input, "table-index %d", &table_index))
3036         table_index_set = 1;
3037       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3038                          vnet_get_main (), &sw_if_index))
3039         ;
3040       else
3041         break;
3042     }
3043
3044   if (table_index_set == 0)
3045     return clib_error_return (0, "classify table-index must be specified");
3046
3047   if (sw_if_index == ~0)
3048     return clib_error_return (0, "interface / subif must be specified");
3049
3050   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3051
3052   switch (rv)
3053     {
3054     case 0:
3055       break;
3056
3057     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3058       return clib_error_return (0, "No such interface");
3059
3060     case VNET_API_ERROR_NO_SUCH_ENTRY:
3061       return clib_error_return (0, "No such classifier table");
3062     }
3063   return 0;
3064 }
3065
3066 /*?
3067  * Assign a classification table to an interface. The classification
3068  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3069  * commands. Once the table is create, use this command to filter packets
3070  * on an interface.
3071  *
3072  * @cliexpar
3073  * Example of how to assign a classification table to an interface:
3074  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3075 ?*/
3076 /* *INDENT-OFF* */
3077 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3078 {
3079   .path = "set ip6 classify",
3080   .short_help =
3081   "set ip6 classify intfc <interface> table-index <classify-idx>",
3082   .function = set_ip6_classify_command_fn,
3083 };
3084 /* *INDENT-ON* */
3085
3086 static clib_error_t *
3087 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3088 {
3089   ip6_main_t *im = &ip6_main;
3090   uword heapsize = 0;
3091   u32 tmp;
3092   u32 nbuckets = 0;
3093
3094   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3095     {
3096       if (unformat (input, "hash-buckets %d", &tmp))
3097         nbuckets = tmp;
3098       else if (unformat (input, "heap-size %U",
3099                          unformat_memory_size, &heapsize))
3100         ;
3101       else
3102         return clib_error_return (0, "unknown input '%U'",
3103                                   format_unformat_error, input);
3104     }
3105
3106   im->lookup_table_nbuckets = nbuckets;
3107   im->lookup_table_size = heapsize;
3108
3109   return 0;
3110 }
3111
3112 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3113
3114 /*
3115  * fd.io coding-style-patch-verification: ON
3116  *
3117  * Local Variables:
3118  * eval: (c-set-style "gnu")
3119  * End:
3120  */