src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_link.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52 #include <vnet/classify/vnet_classify.h>
  53 #include <vnet/pg/pg.h>
  54
  55 #ifndef CLIB_MARCH_VARIANT
  56 #include <vppinfra/bihash_template.c>
  57 #endif
  58 #include <vnet/ip/ip6_forward.h>
  59 #include <vnet/interface_output.h>
  60
  61 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  62 #define OI_DECAP   0x80000000
  63
  64 static void
  65 ip6_add_interface_prefix_routes (ip6_main_t * im,
  66                                  u32 sw_if_index,
  67                                  u32 fib_index,
  68                                  ip6_address_t * address, u32 address_length)
  69 {
  70   ip_lookup_main_t *lm = &im->lookup_main;
  71   ip_interface_prefix_t *if_prefix;
  72
  73   /* *INDENT-OFF* */
  74   ip_interface_prefix_key_t key = {
  75     .prefix = {
  76       .fp_len = address_length,
  77       .fp_proto = FIB_PROTOCOL_IP6,
  78       .fp_addr.ip6 = {
  79         .as_u64 = {
  80           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
  81           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
  82         },
  83       },
  84     },
  85     .sw_if_index = sw_if_index,
  86   };
  87   /* *INDENT-ON* */
  88
  89   /* If prefix already set on interface, just increment ref count & return */
  90   if_prefix = ip_get_interface_prefix (lm, &key);
  91   if (if_prefix)
  92     {
  93       if_prefix->ref_count += 1;
  94       return;
  95     }
  96
  97   /* New prefix - allocate a pool entry, initialize it, add to the hash */
  98   pool_get (lm->if_prefix_pool, if_prefix);
  99   if_prefix->ref_count = 1;
 100   clib_memcpy (&if_prefix->key, &key, sizeof (key));
 101   mhash_set (&lm->prefix_to_if_prefix_index, &key,
 102              if_prefix - lm->if_prefix_pool, 0 /* old value */ );
 103
 104   /* length < 128 - add glean */
 105   if (address_length < 128)
 106     {
 107       /* set the glean route for the prefix */
 108       fib_table_entry_update_one_path (fib_index, &key.prefix,
 109                                        FIB_SOURCE_INTERFACE,
 110                                        (FIB_ENTRY_FLAG_CONNECTED |
 111                                         FIB_ENTRY_FLAG_ATTACHED),
 112                                        DPO_PROTO_IP6,
 113                                        /* No next-hop address */
 114                                        NULL, sw_if_index,
 115                                        /* invalid FIB index */
 116                                        ~0, 1,
 117                                        /* no out-label stack */
 118                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 119     }
 120 }
 121
 122 static void
 123 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 124                           ip6_main_t * im, u32 fib_index,
 125                           ip_interface_address_t * a)
 126 {
 127   ip_lookup_main_t *lm = &im->lookup_main;
 128   ip6_address_t *address = ip_interface_address_get_address (lm, a);
 129   fib_prefix_t pfx = {
 130     .fp_len = a->address_length,
 131     .fp_proto = FIB_PROTOCOL_IP6,
 132     .fp_addr.ip6 = *address,
 133   };
 134
 135   /* set special routes for the prefix if needed */
 136   ip6_add_interface_prefix_routes (im, sw_if_index, fib_index,
 137                                    address, a->address_length);
 138
 139   pfx.fp_len = 128;
 140   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 141     {
 142       u32 classify_table_index =
 143         lm->classify_table_index_by_sw_if_index[sw_if_index];
 144       if (classify_table_index != (u32) ~ 0)
 145         {
 146           dpo_id_t dpo = DPO_INVALID;
 147
 148           dpo_set (&dpo,
 149                    DPO_CLASSIFY,
 150                    DPO_PROTO_IP6,
 151                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 152
 153           fib_table_entry_special_dpo_add (fib_index,
 154                                            &pfx,
 155                                            FIB_SOURCE_CLASSIFY,
 156                                            FIB_ENTRY_FLAG_NONE, &dpo);
 157           dpo_reset (&dpo);
 158         }
 159     }
 160
 161   fib_table_entry_update_one_path (fib_index, &pfx,
 162                                    FIB_SOURCE_INTERFACE,
 163                                    (FIB_ENTRY_FLAG_CONNECTED |
 164                                     FIB_ENTRY_FLAG_LOCAL),
 165                                    DPO_PROTO_IP6,
 166                                    &pfx.fp_addr,
 167                                    sw_if_index, ~0,
 168                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 169 }
 170
 171 static void
 172 ip6_del_interface_prefix_routes (ip6_main_t * im,
 173                                  u32 sw_if_index,
 174                                  u32 fib_index,
 175                                  ip6_address_t * address, u32 address_length)
 176 {
 177   ip_lookup_main_t *lm = &im->lookup_main;
 178   ip_interface_prefix_t *if_prefix;
 179
 180   /* *INDENT-OFF* */
 181   ip_interface_prefix_key_t key = {
 182     .prefix = {
 183       .fp_len = address_length,
 184       .fp_proto = FIB_PROTOCOL_IP6,
 185       .fp_addr.ip6 = {
 186         .as_u64 = {
 187           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
 188           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
 189         },
 190       },
 191     },
 192     .sw_if_index = sw_if_index,
 193   };
 194   /* *INDENT-ON* */
 195
 196   if_prefix = ip_get_interface_prefix (lm, &key);
 197   if (!if_prefix)
 198     {
 199       clib_warning ("Prefix not found while deleting %U",
 200                     format_ip4_address_and_length, address, address_length);
 201       return;
 202     }
 203
 204   /* If not deleting last intf addr in prefix, decrement ref count & return */
 205   if_prefix->ref_count -= 1;
 206   if (if_prefix->ref_count > 0)
 207     return;
 208
 209   /* length <= 128, delete glean route */
 210   if (address_length <= 128)
 211     {
 212       /* remove glean route for prefix */
 213       fib_table_entry_delete (fib_index, &key.prefix, FIB_SOURCE_INTERFACE);
 214     }
 215
 216   mhash_unset (&lm->prefix_to_if_prefix_index, &key, 0 /* old_value */ );
 217   pool_put (lm->if_prefix_pool, if_prefix);
 218 }
 219
 220 static void
 221 ip6_del_interface_routes (u32 sw_if_index, ip6_main_t * im,
 222                           u32 fib_index,
 223                           ip6_address_t * address, u32 address_length)
 224 {
 225   fib_prefix_t pfx = {
 226     .fp_len = 128,
 227     .fp_proto = FIB_PROTOCOL_IP6,
 228     .fp_addr.ip6 = *address,
 229   };
 230
 231   /* delete special routes for the prefix if needed */
 232   ip6_del_interface_prefix_routes (im, sw_if_index, fib_index,
 233                                    address, address_length);
 234
 235   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 236 }
 237
 238 #ifndef CLIB_MARCH_VARIANT
 239 void
 240 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 241 {
 242   ip6_main_t *im = &ip6_main;
 243   vnet_main_t *vnm = vnet_get_main ();
 244   vnet_hw_interface_t *hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
 245
 246   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 247
 248   /*
 249    * enable/disable only on the 1<->0 transition
 250    */
 251   if (is_enable)
 252     {
 253       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 254         return;
 255     }
 256   else
 257     {
 258       /* The ref count is 0 when an address is removed from an interface that has
 259        * no address - this is not a ciritical error */
 260       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 261           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 262         return;
 263     }
 264
 265   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 266                                !is_enable, 0, 0);
 267
 268   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 269                                sw_if_index, !is_enable, 0, 0);
 270
 271   if (is_enable)
 272     hi->l3_if_count++;
 273   else if (hi->l3_if_count)
 274     hi->l3_if_count--;
 275 }
 276
 277 /* get first interface address */
 278 ip6_address_t *
 279 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 280 {
 281   ip_lookup_main_t *lm = &im->lookup_main;
 282   ip_interface_address_t *ia = 0;
 283   ip6_address_t *result = 0;
 284
 285   /* *INDENT-OFF* */
 286   foreach_ip_interface_address (lm, ia, sw_if_index,
 287                                 1 /* honor unnumbered */,
 288   ({
 289     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 290     result = a;
 291     break;
 292   }));
 293   /* *INDENT-ON* */
 294   return result;
 295 }
 296
 297 clib_error_t *
 298 ip6_add_del_interface_address (vlib_main_t * vm,
 299                                u32 sw_if_index,
 300                                ip6_address_t * address,
 301                                u32 address_length, u32 is_del)
 302 {
 303   vnet_main_t *vnm = vnet_get_main ();
 304   ip6_main_t *im = &ip6_main;
 305   ip_lookup_main_t *lm = &im->lookup_main;
 306   clib_error_t *error = NULL;
 307   u32 if_address_index;
 308   ip6_address_fib_t ip6_af, *addr_fib = 0;
 309   const ip6_address_t *ll_addr;
 310
 311   /* local0 interface doesn't support IP addressing */
 312   if (sw_if_index == 0)
 313     {
 314       return
 315         clib_error_create ("local0 interface doesn't support IP addressing");
 316     }
 317
 318   if (ip6_address_is_link_local_unicast (address))
 319     {
 320       if (address_length != 128)
 321         {
 322           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 323           return
 324             clib_error_create
 325             ("prefix length of link-local address must be 128");
 326         }
 327       if (!is_del)
 328         {
 329           int rv;
 330
 331           rv = ip6_link_set_local_address (sw_if_index, address);
 332
 333           if (rv)
 334             {
 335               vnm->api_errno = rv;
 336               return clib_error_create ("address not assignable");
 337             }
 338         }
 339       else
 340         {
 341           ll_addr = ip6_get_link_local_address (sw_if_index);
 342           if (ip6_address_is_equal (ll_addr, address))
 343             {
 344               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 345               return clib_error_create ("address not deletable");
 346             }
 347           else
 348             {
 349               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 350               return clib_error_create ("address not found");
 351             }
 352         }
 353
 354       return (NULL);
 355     }
 356
 357   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 358   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 359
 360   ip6_addr_fib_init (&ip6_af, address,
 361                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 362   vec_add1 (addr_fib, ip6_af);
 363
 364   /* *INDENT-OFF* */
 365   if (!is_del)
 366     {
 367       /* When adding an address check that it does not conflict
 368          with an existing address on any interface in this table. */
 369       ip_interface_address_t *ia;
 370       vnet_sw_interface_t *sif;
 371
 372       pool_foreach (sif, vnm->interface_main.sw_interfaces)
 373        {
 374           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 375               im->fib_index_by_sw_if_index[sif->sw_if_index])
 376             {
 377               foreach_ip_interface_address
 378                 (&im->lookup_main, ia, sif->sw_if_index,
 379                  0 /* honor unnumbered */ ,
 380                  ({
 381                    ip6_address_t * x =
 382                      ip_interface_address_get_address
 383                      (&im->lookup_main, ia);
 384
 385                    if (ip6_destination_matches_route
 386                        (im, address, x, ia->address_length) ||
 387                        ip6_destination_matches_route (im,
 388                                                       x,
 389                                                       address,
 390                                                       address_length))
 391                      {
 392                        /* an intf may have >1 addr from the same prefix */
 393                        if ((sw_if_index == sif->sw_if_index) &&
 394                            (ia->address_length == address_length) &&
 395                            !ip6_address_is_equal (x, address))
 396                          continue;
 397
 398                        if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
 399                          /* if the address we're comparing against is stale
 400                           * then the CP has not added this one back yet, maybe
 401                           * it never will, so we have to assume it won't and
 402                           * ignore it. if it does add it back, then it will fail
 403                           * because this one is now present */
 404                          continue;
 405
 406                        /* error if the length or intf was different */
 407                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 408                        error =  clib_error_create
 409                          ("failed to add %U which conflicts with %U for interface %U",
 410                           format_ip6_address_and_length, address,
 411                           address_length,
 412                           format_ip6_address_and_length, x,
 413                           ia->address_length,
 414                           format_vnet_sw_if_index_name, vnm,
 415                           sif->sw_if_index);
 416                        goto done;
 417                      }
 418                  }));
 419             }
 420       }
 421     }
 422   /* *INDENT-ON* */
 423
 424   if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
 425
 426   if (is_del)
 427     {
 428       if (~0 == if_address_index)
 429         {
 430           vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 431           error = clib_error_create ("%U not found for interface %U",
 432                                      lm->format_address_and_length,
 433                                      addr_fib, address_length,
 434                                      format_vnet_sw_if_index_name, vnm,
 435                                      sw_if_index);
 436           goto done;
 437         }
 438
 439       error = ip_interface_address_del (lm, vnm, if_address_index, addr_fib,
 440                                         address_length, sw_if_index);
 441       if (error)
 442         goto done;
 443     }
 444   else
 445     {
 446       if (~0 != if_address_index)
 447         {
 448           ip_interface_address_t *ia;
 449
 450           ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
 451
 452           if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
 453             {
 454               if (ia->sw_if_index == sw_if_index)
 455                 {
 456                   /* re-adding an address during the replace action.
 457                    * consdier this the update. clear the flag and
 458                    * we're done */
 459                   ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
 460                   goto done;
 461                 }
 462               else
 463                 {
 464                   /* The prefix is moving from one interface to another.
 465                    * delete the stale and add the new */
 466                   ip6_add_del_interface_address (vm,
 467                                                  ia->sw_if_index,
 468                                                  address, address_length, 1);
 469                   ia = NULL;
 470                   error = ip_interface_address_add (lm, sw_if_index,
 471                                                     addr_fib, address_length,
 472                                                     &if_address_index);
 473                 }
 474             }
 475           else
 476             {
 477               vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 478               error = clib_error_create
 479                 ("Prefix %U already found on interface %U",
 480                  lm->format_address_and_length, addr_fib, address_length,
 481                  format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
 482             }
 483         }
 484       else
 485         error = ip_interface_address_add (lm, sw_if_index,
 486                                           addr_fib, address_length,
 487                                           &if_address_index);
 488     }
 489
 490   if (error)
 491     goto done;
 492
 493   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 494   if (!is_del)
 495     ip6_link_enable (sw_if_index, NULL);
 496
 497   /* intf addr routes are added/deleted on admin up/down */
 498   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index))
 499     {
 500       if (is_del)
 501         ip6_del_interface_routes (sw_if_index,
 502                                   im, ip6_af.fib_index, address,
 503                                   address_length);
 504       else
 505         ip6_add_interface_routes (vnm, sw_if_index,
 506                                   im, ip6_af.fib_index,
 507                                   pool_elt_at_index (lm->if_address_pool,
 508                                                      if_address_index));
 509     }
 510
 511   ip6_add_del_interface_address_callback_t *cb;
 512   vec_foreach (cb, im->add_del_interface_address_callbacks)
 513     cb->function (im, cb->function_opaque, sw_if_index,
 514                   address, address_length, if_address_index, is_del);
 515
 516   if (is_del)
 517     ip6_link_disable (sw_if_index);
 518
 519 done:
 520   vec_free (addr_fib);
 521   return error;
 522 }
 523
 524 #endif
 525
 526 static clib_error_t *
 527 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 528 {
 529   ip6_main_t *im = &ip6_main;
 530   ip_interface_address_t *ia;
 531   ip6_address_t *a;
 532   u32 is_admin_up, fib_index;
 533
 534   /* Fill in lookup tables with default table (0). */
 535   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 536
 537   vec_validate_init_empty (im->
 538                            lookup_main.if_address_pool_index_by_sw_if_index,
 539                            sw_if_index, ~0);
 540
 541   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 542
 543   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 544
 545   /* *INDENT-OFF* */
 546   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 547                                 0 /* honor unnumbered */,
 548   ({
 549     a = ip_interface_address_get_address (&im->lookup_main, ia);
 550     if (is_admin_up)
 551       ip6_add_interface_routes (vnm, sw_if_index,
 552                                 im, fib_index,
 553                                 ia);
 554     else
 555       ip6_del_interface_routes (sw_if_index, im, fib_index,
 556                                 a, ia->address_length);
 557   }));
 558   /* *INDENT-ON* */
 559
 560   return 0;
 561 }
 562
 563 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 564
 565 /* Built-in ip6 unicast rx feature path definition */
 566 /* *INDENT-OFF* */
 567 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 568 {
 569   .arc_name  = "ip6-unicast",
 570   .start_nodes = VNET_FEATURES ("ip6-input"),
 571   .last_in_arc = "ip6-lookup",
 572   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 573 };
 574
 575 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 576 {
 577   .arc_name = "ip6-unicast",
 578   .node_name = "ip6-flow-classify",
 579   .runs_before = VNET_FEATURES ("ip6-inacl"),
 580 };
 581
 582 VNET_FEATURE_INIT (ip6_inacl, static) =
 583 {
 584   .arc_name = "ip6-unicast",
 585   .node_name = "ip6-inacl",
 586   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 587 };
 588
 589 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 590 {
 591   .arc_name = "ip6-unicast",
 592   .node_name = "ip6-policer-classify",
 593   .runs_before = VNET_FEATURES ("ipsec6-input-feature"),
 594 };
 595
 596 VNET_FEATURE_INIT (ip6_ipsec, static) =
 597 {
 598   .arc_name = "ip6-unicast",
 599   .node_name = "ipsec6-input-feature",
 600   .runs_before = VNET_FEATURES ("l2tp-decap"),
 601 };
 602
 603 VNET_FEATURE_INIT (ip6_l2tp, static) =
 604 {
 605   .arc_name = "ip6-unicast",
 606   .node_name = "l2tp-decap",
 607   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 608 };
 609
 610 VNET_FEATURE_INIT (ip6_vpath, static) =
 611 {
 612   .arc_name = "ip6-unicast",
 613   .node_name = "vpath-input-ip6",
 614   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 615 };
 616
 617 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 618 {
 619   .arc_name = "ip6-unicast",
 620   .node_name = "ip6-vxlan-bypass",
 621   .runs_before = VNET_FEATURES ("ip6-lookup"),
 622 };
 623
 624 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 625 {
 626   .arc_name = "ip6-unicast",
 627   .node_name = "ip6-not-enabled",
 628   .runs_before = VNET_FEATURES ("ip6-lookup"),
 629 };
 630
 631 VNET_FEATURE_INIT (ip6_lookup, static) =
 632 {
 633   .arc_name = "ip6-unicast",
 634   .node_name = "ip6-lookup",
 635   .runs_before = 0,  /*last feature*/
 636 };
 637
 638 /* Built-in ip6 multicast rx feature path definition (none now) */
 639 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 640 {
 641   .arc_name  = "ip6-multicast",
 642   .start_nodes = VNET_FEATURES ("ip6-input"),
 643   .last_in_arc = "ip6-mfib-forward-lookup",
 644   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 645 };
 646
 647 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 648   .arc_name = "ip6-multicast",
 649   .node_name = "vpath-input-ip6",
 650   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 651 };
 652
 653 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 654   .arc_name = "ip6-multicast",
 655   .node_name = "ip6-not-enabled",
 656   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 657 };
 658
 659 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 660   .arc_name = "ip6-multicast",
 661   .node_name = "ip6-mfib-forward-lookup",
 662   .runs_before = 0, /* last feature */
 663 };
 664
 665 /* Built-in ip4 tx feature path definition */
 666 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 667 {
 668   .arc_name  = "ip6-output",
 669   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 670   .last_in_arc = "interface-output",
 671   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 672 };
 673
 674 VNET_FEATURE_INIT (ip6_outacl, static) = {
 675   .arc_name = "ip6-output",
 676   .node_name = "ip6-outacl",
 677   .runs_before = VNET_FEATURES ("ipsec6-output-feature"),
 678 };
 679
 680 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 681   .arc_name = "ip6-output",
 682   .node_name = "ipsec6-output-feature",
 683   .runs_before = VNET_FEATURES ("interface-output"),
 684 };
 685
 686 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 687   .arc_name = "ip6-output",
 688   .node_name = "interface-output",
 689   .runs_before = 0, /* not before any other features */
 690 };
 691 /* *INDENT-ON* */
 692
 693 static clib_error_t *
 694 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 695 {
 696   ip6_main_t *im = &ip6_main;
 697
 698   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 699   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 700
 701   if (!is_add)
 702     {
 703       /* Ensure that IPv6 is disabled */
 704       ip6_main_t *im6 = &ip6_main;
 705       ip_lookup_main_t *lm6 = &im6->lookup_main;
 706       ip_interface_address_t *ia = 0;
 707       ip6_address_t *address;
 708       vlib_main_t *vm = vlib_get_main ();
 709
 710       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 711       /* *INDENT-OFF* */
 712       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 713       ({
 714         address = ip_interface_address_get_address (lm6, ia);
 715         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 716       }));
 717       /* *INDENT-ON* */
 718       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 719     }
 720
 721   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 722                                is_add, 0, 0);
 723
 724   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 725                                sw_if_index, is_add, 0, 0);
 726
 727   return /* no error */ 0;
 728 }
 729
 730 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 731
 732 VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 733                                 vlib_node_runtime_t * node,
 734                                 vlib_frame_t * frame)
 735 {
 736   return ip6_lookup_inline (vm, node, frame);
 737 }
 738
 739 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 740
 741 /* *INDENT-OFF* */
 742 VLIB_REGISTER_NODE (ip6_lookup_node) =
 743 {
 744   .name = "ip6-lookup",
 745   .vector_size = sizeof (u32),
 746   .format_trace = format_ip6_lookup_trace,
 747   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 748   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 749 };
 750 /* *INDENT-ON* */
 751
 752 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
 753                                       vlib_node_runtime_t * node,
 754                                       vlib_frame_t * frame)
 755 {
 756   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 757   u32 n_left, *from;
 758   u32 thread_index = vm->thread_index;
 759   ip6_main_t *im = &ip6_main;
 760   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 761   u16 nexts[VLIB_FRAME_SIZE], *next;
 762
 763   from = vlib_frame_vector_args (frame);
 764   n_left = frame->n_vectors;
 765   next = nexts;
 766
 767   vlib_get_buffers (vm, from, bufs, n_left);
 768
 769   while (n_left >= 4)
 770     {
 771       const load_balance_t *lb0, *lb1;
 772       const ip6_header_t *ip0, *ip1;
 773       u32 lbi0, hc0, lbi1, hc1;
 774       const dpo_id_t *dpo0, *dpo1;
 775
 776       /* Prefetch next iteration. */
 777       {
 778         vlib_prefetch_buffer_header (b[2], STORE);
 779         vlib_prefetch_buffer_header (b[3], STORE);
 780
 781         CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), STORE);
 782         CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), STORE);
 783       }
 784
 785       ip0 = vlib_buffer_get_current (b[0]);
 786       ip1 = vlib_buffer_get_current (b[1]);
 787       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 788       lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
 789
 790       lb0 = load_balance_get (lbi0);
 791       lb1 = load_balance_get (lbi1);
 792
 793       /*
 794        * this node is for via FIBs we can re-use the hash value from the
 795        * to node if present.
 796        * We don't want to use the same hash value at each level in the recursion
 797        * graph as that would lead to polarisation
 798        */
 799       hc0 = hc1 = 0;
 800
 801       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 802         {
 803           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 804             {
 805               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 806                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 807             }
 808           else
 809             {
 810               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 811                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 812             }
 813           dpo0 = load_balance_get_fwd_bucket
 814             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 815         }
 816       else
 817         {
 818           dpo0 = load_balance_get_bucket_i (lb0, 0);
 819         }
 820       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 821         {
 822           if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash))
 823             {
 824               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 825                 vnet_buffer (b[1])->ip.flow_hash >> 1;
 826             }
 827           else
 828             {
 829               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 830                 ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 831             }
 832           dpo1 = load_balance_get_fwd_bucket
 833             (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 834         }
 835       else
 836         {
 837           dpo1 = load_balance_get_bucket_i (lb1, 0);
 838         }
 839
 840       next[0] = dpo0->dpoi_next_node;
 841       next[1] = dpo1->dpoi_next_node;
 842
 843       /* Only process the HBH Option Header if explicitly configured to do so */
 844       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 845         {
 846           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 847             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 848         }
 849       /* Only process the HBH Option Header if explicitly configured to do so */
 850       if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 851         {
 852           next[1] = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 853             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[1];
 854         }
 855
 856       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 857       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 858
 859       vlib_increment_combined_counter
 860         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 861       vlib_increment_combined_counter
 862         (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1]));
 863
 864       b += 2;
 865       next += 2;
 866       n_left -= 2;
 867     }
 868
 869   while (n_left > 0)
 870     {
 871       const load_balance_t *lb0;
 872       const ip6_header_t *ip0;
 873       const dpo_id_t *dpo0;
 874       u32 lbi0, hc0;
 875
 876       ip0 = vlib_buffer_get_current (b[0]);
 877       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 878
 879       lb0 = load_balance_get (lbi0);
 880
 881       hc0 = 0;
 882       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 883         {
 884           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 885             {
 886               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 887                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 888             }
 889           else
 890             {
 891               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 892                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 893             }
 894           dpo0 = load_balance_get_fwd_bucket
 895             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 896         }
 897       else
 898         {
 899           dpo0 = load_balance_get_bucket_i (lb0, 0);
 900         }
 901
 902       next[0] = dpo0->dpoi_next_node;
 903       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 904
 905       /* Only process the HBH Option Header if explicitly configured to do so */
 906       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 907         {
 908           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 909             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 910         }
 911
 912       vlib_increment_combined_counter
 913         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 914
 915       b += 1;
 916       next += 1;
 917       n_left -= 1;
 918     }
 919
 920   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
 921
 922   if (node->flags & VLIB_NODE_FLAG_TRACE)
 923     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 924
 925   return frame->n_vectors;
 926 }
 927
 928 /* *INDENT-OFF* */
 929 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 930 {
 931   .name = "ip6-load-balance",
 932   .vector_size = sizeof (u32),
 933   .sibling_of = "ip6-lookup",
 934   .format_trace = format_ip6_lookup_trace,
 935 };
 936 /* *INDENT-ON* */
 937
 938 typedef struct
 939 {
 940   /* Adjacency taken. */
 941   u32 adj_index;
 942   u32 flow_hash;
 943   u32 fib_index;
 944
 945   /* Packet data, possibly *after* rewrite. */
 946   u8 packet_data[128 - 1 * sizeof (u32)];
 947 }
 948 ip6_forward_next_trace_t;
 949
 950 #ifndef CLIB_MARCH_VARIANT
 951 u8 *
 952 format_ip6_forward_next_trace (u8 * s, va_list * args)
 953 {
 954   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 955   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 956   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 957   u32 indent = format_get_indent (s);
 958
 959   s = format (s, "%Ufib:%d adj:%d flow:%d",
 960               format_white_space, indent,
 961               t->fib_index, t->adj_index, t->flow_hash);
 962   s = format (s, "\n%U%U",
 963               format_white_space, indent,
 964               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 965   return s;
 966 }
 967 #endif
 968
 969 static u8 *
 970 format_ip6_lookup_trace (u8 * s, va_list * args)
 971 {
 972   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 973   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 974   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 975   u32 indent = format_get_indent (s);
 976
 977   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 978               t->fib_index, t->adj_index, t->flow_hash);
 979   s = format (s, "\n%U%U",
 980               format_white_space, indent,
 981               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 982   return s;
 983 }
 984
 985
 986 static u8 *
 987 format_ip6_rewrite_trace (u8 * s, va_list * args)
 988 {
 989   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 990   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 991   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 992   u32 indent = format_get_indent (s);
 993
 994   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 995               t->fib_index, t->adj_index, format_ip_adjacency,
 996               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 997   s = format (s, "\n%U%U",
 998               format_white_space, indent,
 999               format_ip_adjacency_packet_data,
1000               t->packet_data, sizeof (t->packet_data));
1001   return s;
1002 }
1003
1004 /* Common trace function for all ip6-forward next nodes. */
1005 #ifndef CLIB_MARCH_VARIANT
1006 void
1007 ip6_forward_next_trace (vlib_main_t * vm,
1008                         vlib_node_runtime_t * node,
1009                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1010 {
1011   u32 *from, n_left;
1012   ip6_main_t *im = &ip6_main;
1013
1014   n_left = frame->n_vectors;
1015   from = vlib_frame_vector_args (frame);
1016
1017   while (n_left >= 4)
1018     {
1019       u32 bi0, bi1;
1020       vlib_buffer_t *b0, *b1;
1021       ip6_forward_next_trace_t *t0, *t1;
1022
1023       /* Prefetch next iteration. */
1024       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1025       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1026
1027       bi0 = from[0];
1028       bi1 = from[1];
1029
1030       b0 = vlib_get_buffer (vm, bi0);
1031       b1 = vlib_get_buffer (vm, bi1);
1032
1033       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1034         {
1035           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1036           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1037           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1038           t0->fib_index =
1039             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1040              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1041             vec_elt (im->fib_index_by_sw_if_index,
1042                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1043
1044           clib_memcpy_fast (t0->packet_data,
1045                             vlib_buffer_get_current (b0),
1046                             sizeof (t0->packet_data));
1047         }
1048       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1049         {
1050           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1051           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1052           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1053           t1->fib_index =
1054             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1055              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1056             vec_elt (im->fib_index_by_sw_if_index,
1057                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1058
1059           clib_memcpy_fast (t1->packet_data,
1060                             vlib_buffer_get_current (b1),
1061                             sizeof (t1->packet_data));
1062         }
1063       from += 2;
1064       n_left -= 2;
1065     }
1066
1067   while (n_left >= 1)
1068     {
1069       u32 bi0;
1070       vlib_buffer_t *b0;
1071       ip6_forward_next_trace_t *t0;
1072
1073       bi0 = from[0];
1074
1075       b0 = vlib_get_buffer (vm, bi0);
1076
1077       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1078         {
1079           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1080           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1081           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1082           t0->fib_index =
1083             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1084              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1085             vec_elt (im->fib_index_by_sw_if_index,
1086                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1087
1088           clib_memcpy_fast (t0->packet_data,
1089                             vlib_buffer_get_current (b0),
1090                             sizeof (t0->packet_data));
1091         }
1092       from += 1;
1093       n_left -= 1;
1094     }
1095 }
1096
1097 /* Compute TCP/UDP/ICMP6 checksum in software. */
1098 u16
1099 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1100                                    ip6_header_t * ip0, int *bogus_lengthp)
1101 {
1102   ip_csum_t sum0 = 0;
1103   u16 payload_length, payload_length_host_byte_order;
1104   u32 i;
1105   u32 headers_size = sizeof (ip0[0]);
1106   u8 *data_this_buffer;
1107   u8 next_hdr = ip0->protocol;
1108
1109   ASSERT (bogus_lengthp);
1110   *bogus_lengthp = 0;
1111
1112   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1113   data_this_buffer = (u8 *) (ip0 + 1);
1114   payload_length = ip0->payload_length;
1115
1116   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
1117    * or UDP-Ping packets */
1118   if (PREDICT_FALSE (next_hdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1119     {
1120       u32 skip_bytes;
1121       ip6_hop_by_hop_ext_t *ext_hdr =
1122         (ip6_hop_by_hop_ext_t *) data_this_buffer;
1123
1124       /* validate really icmp6 next */
1125       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1126               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
1127
1128       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
1129       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
1130
1131       payload_length_host_byte_order -= skip_bytes;
1132       headers_size += skip_bytes;
1133
1134       /* pseudo-header adjustments:
1135        *   exclude ext header bytes from payload length
1136        *   use payload IP proto rather than ext header IP proto
1137        */
1138       payload_length = clib_host_to_net_u16 (payload_length_host_byte_order);
1139       next_hdr = ext_hdr->next_hdr;
1140     }
1141
1142   /* Initialize checksum with ip pseudo-header. */
1143   sum0 = payload_length + clib_host_to_net_u16 (next_hdr);
1144
1145   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1146     {
1147       sum0 = ip_csum_with_carry
1148         (sum0, clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1149       sum0 = ip_csum_with_carry
1150         (sum0, clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1151     }
1152
1153   if (p0)
1154     return ip_calculate_l4_checksum (vm, p0, sum0,
1155                                      payload_length_host_byte_order,
1156                                      (u8 *) ip0, headers_size, NULL);
1157   else
1158     return ip_calculate_l4_checksum (vm, 0, sum0,
1159                                      payload_length_host_byte_order, NULL, 0,
1160                                      data_this_buffer);
1161 }
1162
1163 u32
1164 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1165 {
1166   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1167   udp_header_t *udp0;
1168   u16 sum16;
1169   int bogus_length;
1170
1171   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1172   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1173           || ip0->protocol == IP_PROTOCOL_ICMP6
1174           || ip0->protocol == IP_PROTOCOL_UDP
1175           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1176
1177   udp0 = (void *) (ip0 + 1);
1178   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1179     {
1180       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1181                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1182       return p0->flags;
1183     }
1184
1185   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1186
1187   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1188                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1189
1190   return p0->flags;
1191 }
1192 #endif
1193
1194 /**
1195  * @brief returns number of links on which src is reachable.
1196  */
1197 always_inline int
1198 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1199 {
1200   const load_balance_t *lb0;
1201   index_t lbi;
1202   u32 fib_index;
1203
1204   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1205                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1206   fib_index =
1207     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1208     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1209
1210   lbi = ip6_fib_table_fwding_lookup (fib_index, &i->src_address);
1211   lb0 = load_balance_get (lbi);
1212
1213   return (fib_urpf_check_size (lb0->lb_urpf));
1214 }
1215
1216 always_inline u8
1217 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1218                            u32 * udp_offset0)
1219 {
1220   u32 proto0;
1221   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1222   if (proto0 != IP_PROTOCOL_UDP)
1223     {
1224       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1225       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1226     }
1227   return proto0;
1228 }
1229
1230 /* *INDENT-OFF* */
1231 VNET_FEATURE_ARC_INIT (ip6_local) =
1232 {
1233   .arc_name  = "ip6-local",
1234   .start_nodes = VNET_FEATURES ("ip6-local"),
1235 };
1236 /* *INDENT-ON* */
1237
1238 static_always_inline u8
1239 ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
1240 {
1241
1242   u16 payload_length_host_byte_order;
1243   u32 n_this_buffer, n_bytes_left;
1244   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1245   u32 headers_size = sizeof (ip0[0]);
1246   u8 *data_this_buffer;
1247
1248
1249   data_this_buffer = (u8 *) (ip0 + 1);
1250
1251   ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t *) data_this_buffer;
1252
1253   /* validate really icmp6 next */
1254
1255   if (!(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1256       || (ext_hdr->next_hdr == IP_PROTOCOL_UDP))
1257     return 0;
1258
1259
1260   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1261   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1262
1263
1264   u32 n_ip_bytes_this_buffer =
1265     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1266   if (n_this_buffer + headers_size > n_ip_bytes_this_buffer)
1267     {
1268       n_this_buffer = p0->current_length > headers_size ?
1269         n_ip_bytes_this_buffer - headers_size : 0;
1270     }
1271
1272   n_bytes_left -= n_this_buffer;
1273   n_bytes_left -= p0->total_length_not_including_first_buffer;
1274
1275   if (n_bytes_left == 0)
1276     return 0;
1277   else
1278     return 1;
1279 }
1280
1281
1282 always_inline uword
1283 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1284                   vlib_frame_t * frame, int head_of_feature_arc)
1285 {
1286   ip6_main_t *im = &ip6_main;
1287   ip_lookup_main_t *lm = &im->lookup_main;
1288   u32 *from, n_left_from;
1289   vlib_node_runtime_t *error_node =
1290     vlib_node_get_runtime (vm, ip6_input_node.index);
1291   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1292   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1293   u16 nexts[VLIB_FRAME_SIZE], *next;
1294
1295   from = vlib_frame_vector_args (frame);
1296   n_left_from = frame->n_vectors;
1297
1298   if (node->flags & VLIB_NODE_FLAG_TRACE)
1299     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1300
1301   vlib_get_buffers (vm, from, bufs, n_left_from);
1302   b = bufs;
1303   next = nexts;
1304
1305   while (n_left_from > 2)
1306     {
1307       /* Prefetch next iteration. */
1308       if (n_left_from >= 6)
1309         {
1310           vlib_prefetch_buffer_header (b[4], STORE);
1311           vlib_prefetch_buffer_header (b[5], STORE);
1312           vlib_prefetch_buffer_data (b[2], LOAD);
1313           vlib_prefetch_buffer_data (b[3], LOAD);
1314         }
1315
1316       ip6_error_t error[2];
1317       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1318       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1319
1320       ip6_header_t *ip[2];
1321       ip[0] = vlib_buffer_get_current (b[0]);
1322       ip[1] = vlib_buffer_get_current (b[1]);
1323
1324       if (head_of_feature_arc)
1325         {
1326           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1327           vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1328
1329           u8 type[2];
1330           type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol];
1331           type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol];
1332
1333           u32 flags[2];
1334           flags[0] = b[0]->flags;
1335           flags[1] = b[1]->flags;
1336
1337           vnet_buffer_oflags_t oflags[2];
1338           oflags[0] = vnet_buffer (b[0])->oflags;
1339           oflags[1] = vnet_buffer (b[1])->oflags;
1340
1341           u32 l4_offload[2];
1342           l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
1343                           (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
1344                                         VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
1345           l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
1346                           (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
1347                                         VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
1348
1349           u32 good_l4_csum[2];
1350           good_l4_csum[0] =
1351             (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
1352           good_l4_csum[1] =
1353             (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
1354
1355           u32 udp_offset[2] = { };
1356           u8 is_tcp_udp[2];
1357           is_tcp_udp[0] =
1358             ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]);
1359           is_tcp_udp[1] =
1360             ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]);
1361           i16 len_diff[2] = { 0 };
1362           if (PREDICT_TRUE (is_tcp_udp[0]))
1363             {
1364               udp_header_t *udp =
1365                 (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]);
1366               good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP
1367                 && udp->checksum == 0;
1368               /* optimistically verify UDP length. */
1369               u16 ip_len, udp_len;
1370               ip_len = clib_net_to_host_u16 (ip[0]->payload_length);
1371               udp_len = clib_net_to_host_u16 (udp->length);
1372               len_diff[0] = ip_len - udp_len;
1373             }
1374           if (PREDICT_TRUE (is_tcp_udp[1]))
1375             {
1376               udp_header_t *udp =
1377                 (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]);
1378               good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP
1379                 && udp->checksum == 0;
1380               /* optimistically verify UDP length. */
1381               u16 ip_len, udp_len;
1382               ip_len = clib_net_to_host_u16 (ip[1]->payload_length);
1383               udp_len = clib_net_to_host_u16 (udp->length);
1384               len_diff[1] = ip_len - udp_len;
1385             }
1386
1387           good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1388           good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1389
1390           len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0;
1391           len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0;
1392
1393           u8 need_csum[2];
1394           need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN
1395             && !good_l4_csum[0]
1396             && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1397           need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN
1398             && !good_l4_csum[1]
1399             && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1400           if (PREDICT_FALSE (need_csum[0]))
1401             {
1402               flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1403               good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1404               error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1405             }
1406           else
1407             {
1408               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1409                 error[0] = IP6_ERROR_BAD_LENGTH;
1410             }
1411           if (PREDICT_FALSE (need_csum[1]))
1412             {
1413               flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]);
1414               good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1415               error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1416             }
1417           else
1418             {
1419               if (ip6_tcp_udp_icmp_bad_length (vm, b[1]))
1420                 error[1] = IP6_ERROR_BAD_LENGTH;
1421             }
1422
1423
1424           error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0];
1425
1426           error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1];
1427
1428           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1429                          IP6_ERROR_UDP_CHECKSUM,
1430                          "Wrong IP6 errors constants");
1431           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1432                          IP6_ERROR_ICMP_CHECKSUM,
1433                          "Wrong IP6 errors constants");
1434
1435           error[0] =
1436             !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0];
1437           error[1] =
1438             !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1];
1439
1440           /* Drop packets from unroutable hosts. */
1441           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1442           u8 unroutable[2];
1443           unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL
1444             && type[0] != IP_BUILTIN_PROTOCOL_ICMP
1445             && !ip6_address_is_link_local_unicast (&ip[0]->src_address);
1446           unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL
1447             && type[1] != IP_BUILTIN_PROTOCOL_ICMP
1448             && !ip6_address_is_link_local_unicast (&ip[1]->src_address);
1449           if (PREDICT_FALSE (unroutable[0]))
1450             {
1451               error[0] =
1452                 !ip6_urpf_loose_check (im, b[0],
1453                                        ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS
1454                 : error[0];
1455             }
1456           if (PREDICT_FALSE (unroutable[1]))
1457             {
1458               error[1] =
1459                 !ip6_urpf_loose_check (im, b[1],
1460                                        ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS
1461                 : error[1];
1462             }
1463
1464           vnet_buffer (b[0])->ip.fib_index =
1465             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1466             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1467             vnet_buffer (b[0])->ip.fib_index;
1468           vnet_buffer (b[1])->ip.fib_index =
1469             vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1470             vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1471             vnet_buffer (b[1])->ip.fib_index;
1472         }                       /* head_of_feature_arc */
1473
1474       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
1475       next[0] =
1476         error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1477       next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol];
1478       next[1] =
1479         error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1];
1480
1481       b[0]->error = error_node->errors[error[0]];
1482       b[1]->error = error_node->errors[error[1]];
1483
1484       if (head_of_feature_arc)
1485         {
1486           u8 ip6_unknown[2];
1487           ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1488           ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1489           if (PREDICT_TRUE (ip6_unknown[0]))
1490             {
1491               u32 next32 = next[0];
1492               vnet_feature_arc_start (arc_index,
1493                                       vnet_buffer (b[0])->sw_if_index
1494                                       [VLIB_RX], &next32, b[0]);
1495               next[0] = next32;
1496             }
1497           if (PREDICT_TRUE (ip6_unknown[1]))
1498             {
1499               u32 next32 = next[1];
1500               vnet_feature_arc_start (arc_index,
1501                                       vnet_buffer (b[1])->sw_if_index
1502                                       [VLIB_RX], &next32, b[1]);
1503               next[1] = next32;
1504             }
1505         }
1506
1507       /* next */
1508       b += 2;
1509       next += 2;
1510       n_left_from -= 2;
1511     }
1512
1513   while (n_left_from)
1514     {
1515       u8 error;
1516       error = IP6_ERROR_UNKNOWN_PROTOCOL;
1517
1518       ip6_header_t *ip;
1519       ip = vlib_buffer_get_current (b[0]);
1520
1521       if (head_of_feature_arc)
1522         {
1523           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1524           u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
1525
1526           u32 flags = b[0]->flags;
1527
1528           vnet_buffer_oflags_t oflags = vnet_buffer (b[0])->oflags;
1529
1530           u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
1531                            (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
1532                                       VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
1533
1534           u32 good_l4_csum =
1535             (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
1536           u32 udp_offset;
1537           i16 len_diff = 0;
1538           u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
1539           if (PREDICT_TRUE (is_tcp_udp))
1540             {
1541               udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset);
1542               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1543               good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP
1544                 && udp->checksum == 0;
1545               /* optimistically verify UDP length. */
1546               u16 ip_len, udp_len;
1547               ip_len = clib_net_to_host_u16 (ip->payload_length);
1548               udp_len = clib_net_to_host_u16 (udp->length);
1549               len_diff = ip_len - udp_len;
1550             }
1551
1552           good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
1553           len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
1554
1555           u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
1556                          !good_l4_csum &&
1557                          !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1558           if (PREDICT_FALSE (need_csum))
1559             {
1560               flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1561               good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1562               error = IP6_ERROR_UNKNOWN_PROTOCOL;
1563             }
1564           else
1565             {
1566               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1567                 error = IP6_ERROR_BAD_LENGTH;
1568             }
1569
1570
1571
1572           error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
1573           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1574                          IP6_ERROR_UDP_CHECKSUM,
1575                          "Wrong IP6 errors constants");
1576           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1577                          IP6_ERROR_ICMP_CHECKSUM,
1578                          "Wrong IP6 errors constants");
1579
1580           error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error;
1581
1582           /* Drop packets from unroutable hosts. */
1583           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1584           u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL
1585             && type != IP_BUILTIN_PROTOCOL_ICMP
1586             && !ip6_address_is_link_local_unicast (&ip->src_address);
1587           if (PREDICT_FALSE (unroutable))
1588             {
1589               error =
1590                 !ip6_urpf_loose_check (im, b[0],
1591                                        ip) ? IP6_ERROR_SRC_LOOKUP_MISS :
1592                 error;
1593             }
1594
1595           vnet_buffer (b[0])->ip.fib_index =
1596             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1597             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1598             vnet_buffer (b[0])->ip.fib_index;
1599         }                       /* head_of_feature_arc */
1600
1601       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
1602       next[0] =
1603         error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1604
1605       b[0]->error = error_node->errors[error];
1606
1607       if (head_of_feature_arc)
1608         {
1609           if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1610             {
1611               u32 next32 = next[0];
1612               vnet_feature_arc_start (arc_index,
1613                                       vnet_buffer (b[0])->sw_if_index
1614                                       [VLIB_RX], &next32, b[0]);
1615               next[0] = next32;
1616             }
1617         }
1618
1619       /* next */
1620       b += 1;
1621       next += 1;
1622       n_left_from -= 1;
1623     }
1624
1625   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1626   return frame->n_vectors;
1627 }
1628
1629 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1630                                vlib_frame_t * frame)
1631 {
1632   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1633 }
1634
1635 /* *INDENT-OFF* */
1636 VLIB_REGISTER_NODE (ip6_local_node) =
1637 {
1638   .name = "ip6-local",
1639   .vector_size = sizeof (u32),
1640   .format_trace = format_ip6_forward_next_trace,
1641   .n_next_nodes = IP_LOCAL_N_NEXT,
1642   .next_nodes =
1643   {
1644     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1645     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1646     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1647     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1648     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-full-reassembly",
1649   },
1650 };
1651 /* *INDENT-ON* */
1652
1653 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
1654                                           vlib_node_runtime_t * node,
1655                                           vlib_frame_t * frame)
1656 {
1657   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1658 }
1659
1660 /* *INDENT-OFF* */
1661 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
1662   .name = "ip6-local-end-of-arc",
1663   .vector_size = sizeof (u32),
1664
1665   .format_trace = format_ip6_forward_next_trace,
1666   .sibling_of = "ip6-local",
1667 };
1668
1669 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1670   .arc_name = "ip6-local",
1671   .node_name = "ip6-local-end-of-arc",
1672   .runs_before = 0, /* not before any other features */
1673 };
1674 /* *INDENT-ON* */
1675
1676 #ifdef CLIB_MARCH_VARIANT
1677 extern vlib_node_registration_t ip6_local_node;
1678 #else
1679 void
1680 ip6_register_protocol (u32 protocol, u32 node_index)
1681 {
1682   vlib_main_t *vm = vlib_get_main ();
1683   ip6_main_t *im = &ip6_main;
1684   ip_lookup_main_t *lm = &im->lookup_main;
1685
1686   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1687   lm->local_next_by_ip_protocol[protocol] =
1688     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1689 }
1690
1691 void
1692 ip6_unregister_protocol (u32 protocol)
1693 {
1694   ip6_main_t *im = &ip6_main;
1695   ip_lookup_main_t *lm = &im->lookup_main;
1696
1697   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1698   lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT;
1699 }
1700 #endif
1701
1702 typedef enum
1703 {
1704   IP6_REWRITE_NEXT_DROP,
1705   IP6_REWRITE_NEXT_ICMP_ERROR,
1706   IP6_REWRITE_NEXT_FRAGMENT,
1707   IP6_REWRITE_N_NEXT            /* Last */
1708 } ip6_rewrite_next_t;
1709
1710 /**
1711  * This bits of an IPv6 address to mask to construct a multicast
1712  * MAC address
1713  */
1714 #define IP6_MCAST_ADDR_MASK 0xffffffff
1715
1716 always_inline void
1717 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1718                u16 adj_packet_bytes, bool is_locally_generated,
1719                u32 * next, u8 is_midchain, u32 * error)
1720 {
1721   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1722     {
1723       if (is_locally_generated)
1724         {
1725           /* IP fragmentation */
1726           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1727                                    (is_midchain ?
1728                                     IP_FRAG_NEXT_IP_REWRITE_MIDCHAIN :
1729                                     IP_FRAG_NEXT_IP_REWRITE), 0);
1730           *next = IP6_REWRITE_NEXT_FRAGMENT;
1731           *error = IP6_ERROR_MTU_EXCEEDED;
1732         }
1733       else
1734         {
1735           *error = IP6_ERROR_MTU_EXCEEDED;
1736           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1737                                        adj_packet_bytes);
1738           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1739         }
1740     }
1741 }
1742
1743 always_inline uword
1744 ip6_rewrite_inline_with_gso (vlib_main_t * vm,
1745                              vlib_node_runtime_t * node,
1746                              vlib_frame_t * frame,
1747                              int do_counters, int is_midchain, int is_mcast)
1748 {
1749   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1750   u32 *from = vlib_frame_vector_args (frame);
1751   u32 n_left_from, n_left_to_next, *to_next, next_index;
1752   vlib_node_runtime_t *error_node =
1753     vlib_node_get_runtime (vm, ip6_input_node.index);
1754
1755   n_left_from = frame->n_vectors;
1756   next_index = node->cached_next_index;
1757   u32 thread_index = vm->thread_index;
1758
1759   while (n_left_from > 0)
1760     {
1761       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1762
1763       while (n_left_from >= 4 && n_left_to_next >= 2)
1764         {
1765           const ip_adjacency_t *adj0, *adj1;
1766           vlib_buffer_t *p0, *p1;
1767           ip6_header_t *ip0, *ip1;
1768           u32 pi0, rw_len0, next0, error0, adj_index0;
1769           u32 pi1, rw_len1, next1, error1, adj_index1;
1770           u32 tx_sw_if_index0, tx_sw_if_index1;
1771           bool is_locally_originated0, is_locally_originated1;
1772
1773           /* Prefetch next iteration. */
1774           {
1775             vlib_buffer_t *p2, *p3;
1776
1777             p2 = vlib_get_buffer (vm, from[2]);
1778             p3 = vlib_get_buffer (vm, from[3]);
1779
1780             vlib_prefetch_buffer_header (p2, LOAD);
1781             vlib_prefetch_buffer_header (p3, LOAD);
1782
1783             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1784             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1785
1786             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1787             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1788           }
1789
1790           pi0 = to_next[0] = from[0];
1791           pi1 = to_next[1] = from[1];
1792
1793           from += 2;
1794           n_left_from -= 2;
1795           to_next += 2;
1796           n_left_to_next -= 2;
1797
1798           p0 = vlib_get_buffer (vm, pi0);
1799           p1 = vlib_get_buffer (vm, pi1);
1800
1801           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1802           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1803
1804           ip0 = vlib_buffer_get_current (p0);
1805           ip1 = vlib_buffer_get_current (p1);
1806
1807           error0 = error1 = IP6_ERROR_NONE;
1808           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1809
1810           is_locally_originated0 =
1811             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1812           if (PREDICT_TRUE (!is_locally_originated0))
1813             {
1814               i32 hop_limit0 = ip0->hop_limit;
1815
1816               /* Input node should have reject packets with hop limit 0. */
1817               ASSERT (ip0->hop_limit > 0);
1818
1819               hop_limit0 -= 1;
1820
1821               ip0->hop_limit = hop_limit0;
1822
1823               /*
1824                * If the hop count drops below 1 when forwarding, generate
1825                * an ICMP response.
1826                */
1827               if (PREDICT_FALSE (hop_limit0 <= 0))
1828                 {
1829                   error0 = IP6_ERROR_TIME_EXPIRED;
1830                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1831                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1832                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1833                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1834                                                0);
1835                 }
1836             }
1837
1838           is_locally_originated1 =
1839             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1840           if (PREDICT_TRUE (!is_locally_originated1))
1841             {
1842               i32 hop_limit1 = ip1->hop_limit;
1843
1844               /* Input node should have reject packets with hop limit 0. */
1845               ASSERT (ip1->hop_limit > 0);
1846
1847               hop_limit1 -= 1;
1848
1849               ip1->hop_limit = hop_limit1;
1850
1851               /*
1852                * If the hop count drops below 1 when forwarding, generate
1853                * an ICMP response.
1854                */
1855               if (PREDICT_FALSE (hop_limit1 <= 0))
1856                 {
1857                   error1 = IP6_ERROR_TIME_EXPIRED;
1858                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1859                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1860                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1861                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1862                                                0);
1863                 }
1864             }
1865
1866           adj0 = adj_get (adj_index0);
1867           adj1 = adj_get (adj_index1);
1868
1869           rw_len0 = adj0[0].rewrite_header.data_bytes;
1870           rw_len1 = adj1[0].rewrite_header.data_bytes;
1871           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1872           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1873
1874           if (do_counters)
1875             {
1876               vlib_increment_combined_counter
1877                 (&adjacency_counters,
1878                  thread_index, adj_index0, 1,
1879                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1880               vlib_increment_combined_counter
1881                 (&adjacency_counters,
1882                  thread_index, adj_index1, 1,
1883                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1884             }
1885
1886           /* Check MTU of outgoing interface. */
1887           u16 ip0_len =
1888             clib_net_to_host_u16 (ip0->payload_length) +
1889             sizeof (ip6_header_t);
1890           u16 ip1_len =
1891             clib_net_to_host_u16 (ip1->payload_length) +
1892             sizeof (ip6_header_t);
1893           if (p0->flags & VNET_BUFFER_F_GSO)
1894             ip0_len = gso_mtu_sz (p0);
1895           if (p1->flags & VNET_BUFFER_F_GSO)
1896             ip1_len = gso_mtu_sz (p1);
1897
1898           ip6_mtu_check (p0, ip0_len,
1899                          adj0[0].rewrite_header.max_l3_packet_bytes,
1900                          is_locally_originated0, &next0, is_midchain,
1901                          &error0);
1902           ip6_mtu_check (p1, ip1_len,
1903                          adj1[0].rewrite_header.max_l3_packet_bytes,
1904                          is_locally_originated1, &next1, is_midchain,
1905                          &error1);
1906           /* Don't adjust the buffer for hop count issue; icmp-error node
1907            * wants to see the IP header */
1908           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1909             {
1910               p0->current_data -= rw_len0;
1911               p0->current_length += rw_len0;
1912               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1913               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1914               next0 = adj0[0].rewrite_header.next_index;
1915               if (PREDICT_FALSE
1916                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1917                 vnet_feature_arc_start_w_cfg_index
1918                   (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0,
1919                    adj0->ia_cfg_index);
1920             }
1921           else
1922             {
1923               p0->error = error_node->errors[error0];
1924             }
1925           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1926             {
1927               p1->current_data -= rw_len1;
1928               p1->current_length += rw_len1;
1929
1930               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1931               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1932               next1 = adj1[0].rewrite_header.next_index;
1933
1934               if (PREDICT_FALSE
1935                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1936                 vnet_feature_arc_start_w_cfg_index
1937                   (lm->output_feature_arc_index, tx_sw_if_index1, &next1, p1,
1938                    adj1->ia_cfg_index);
1939             }
1940           else
1941             {
1942               p1->error = error_node->errors[error1];
1943             }
1944
1945           if (is_midchain)
1946             {
1947               /* before we paint on the next header, update the L4
1948                * checksums if required, since there's no offload on a tunnel */
1949               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
1950                                           1 /* is_ip6 */ );
1951               vnet_calc_checksums_inline (vm, p1, 0 /* is_ip4 */ ,
1952                                           1 /* is_ip6 */ );
1953
1954               /* Guess we are only writing on ipv6 header. */
1955               vnet_rewrite_two_headers (adj0[0], adj1[0],
1956                                         ip0, ip1, sizeof (ip6_header_t));
1957             }
1958           else
1959             /* Guess we are only writing on simple Ethernet header. */
1960             vnet_rewrite_two_headers (adj0[0], adj1[0],
1961                                       ip0, ip1, sizeof (ethernet_header_t));
1962
1963           if (is_midchain)
1964             {
1965               if (adj0->sub_type.midchain.fixup_func)
1966                 adj0->sub_type.midchain.fixup_func
1967                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1968               if (adj1->sub_type.midchain.fixup_func)
1969                 adj1->sub_type.midchain.fixup_func
1970                   (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1971             }
1972           if (is_mcast)
1973             {
1974               /*
1975                * copy bytes from the IP address into the MAC rewrite
1976                */
1977               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1978                                           adj0->
1979                                           rewrite_header.dst_mcast_offset,
1980                                           &ip0->dst_address.as_u32[3],
1981                                           (u8 *) ip0);
1982               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1983                                           adj1->
1984                                           rewrite_header.dst_mcast_offset,
1985                                           &ip1->dst_address.as_u32[3],
1986                                           (u8 *) ip1);
1987             }
1988
1989           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1990                                            to_next, n_left_to_next,
1991                                            pi0, pi1, next0, next1);
1992         }
1993
1994       while (n_left_from > 0 && n_left_to_next > 0)
1995         {
1996           ip_adjacency_t *adj0;
1997           vlib_buffer_t *p0;
1998           ip6_header_t *ip0;
1999           u32 pi0, rw_len0;
2000           u32 adj_index0, next0, error0;
2001           u32 tx_sw_if_index0;
2002           bool is_locally_originated0;
2003
2004           pi0 = to_next[0] = from[0];
2005
2006           p0 = vlib_get_buffer (vm, pi0);
2007
2008           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2009
2010           adj0 = adj_get (adj_index0);
2011
2012           ip0 = vlib_buffer_get_current (p0);
2013
2014           error0 = IP6_ERROR_NONE;
2015           next0 = IP6_REWRITE_NEXT_DROP;
2016
2017           /* Check hop limit */
2018           is_locally_originated0 =
2019             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
2020           if (PREDICT_TRUE (!is_locally_originated0))
2021             {
2022               i32 hop_limit0 = ip0->hop_limit;
2023
2024               ASSERT (ip0->hop_limit > 0);
2025
2026               hop_limit0 -= 1;
2027
2028               ip0->hop_limit = hop_limit0;
2029
2030               if (PREDICT_FALSE (hop_limit0 <= 0))
2031                 {
2032                   /*
2033                    * If the hop count drops below 1 when forwarding, generate
2034                    * an ICMP response.
2035                    */
2036                   error0 = IP6_ERROR_TIME_EXPIRED;
2037                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2038                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2039                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
2040                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
2041                                                0);
2042                 }
2043             }
2044
2045           if (is_midchain)
2046             {
2047               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
2048                                           1 /* is_ip6 */ );
2049
2050               /* Guess we are only writing on ip6 header. */
2051               vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip6_header_t));
2052             }
2053           else
2054             /* Guess we are only writing on simple Ethernet header. */
2055             vnet_rewrite_one_header (adj0[0], ip0,
2056                                      sizeof (ethernet_header_t));
2057
2058           /* Update packet buffer attributes/set output interface. */
2059           rw_len0 = adj0[0].rewrite_header.data_bytes;
2060           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2061
2062           if (do_counters)
2063             {
2064               vlib_increment_combined_counter
2065                 (&adjacency_counters,
2066                  thread_index, adj_index0, 1,
2067                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2068             }
2069
2070           /* Check MTU of outgoing interface. */
2071           u16 ip0_len =
2072             clib_net_to_host_u16 (ip0->payload_length) +
2073             sizeof (ip6_header_t);
2074           if (p0->flags & VNET_BUFFER_F_GSO)
2075             ip0_len = gso_mtu_sz (p0);
2076
2077           ip6_mtu_check (p0, ip0_len,
2078                          adj0[0].rewrite_header.max_l3_packet_bytes,
2079                          is_locally_originated0, &next0, is_midchain,
2080                          &error0);
2081           /* Don't adjust the buffer for hop count issue; icmp-error node
2082            * wants to see the IP header */
2083           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
2084             {
2085               p0->current_data -= rw_len0;
2086               p0->current_length += rw_len0;
2087
2088               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2089
2090               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2091               next0 = adj0[0].rewrite_header.next_index;
2092
2093               if (PREDICT_FALSE
2094                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2095                 vnet_feature_arc_start_w_cfg_index
2096                   (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0,
2097                    adj0->ia_cfg_index);
2098             }
2099           else
2100             {
2101               p0->error = error_node->errors[error0];
2102             }
2103
2104           if (is_midchain)
2105             {
2106               if (adj0->sub_type.midchain.fixup_func)
2107                 adj0->sub_type.midchain.fixup_func
2108                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2109             }
2110           if (is_mcast)
2111             {
2112               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
2113                                           adj0->
2114                                           rewrite_header.dst_mcast_offset,
2115                                           &ip0->dst_address.as_u32[3],
2116                                           (u8 *) ip0);
2117             }
2118
2119           from += 1;
2120           n_left_from -= 1;
2121           to_next += 1;
2122           n_left_to_next -= 1;
2123
2124           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2125                                            to_next, n_left_to_next,
2126                                            pi0, next0);
2127         }
2128
2129       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2130     }
2131
2132   /* Need to do trace after rewrites to pick up new packet data. */
2133   if (node->flags & VLIB_NODE_FLAG_TRACE)
2134     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2135
2136   return frame->n_vectors;
2137 }
2138
2139 always_inline uword
2140 ip6_rewrite_inline (vlib_main_t * vm,
2141                     vlib_node_runtime_t * node,
2142                     vlib_frame_t * frame,
2143                     int do_counters, int is_midchain, int is_mcast)
2144 {
2145   return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2146                                       is_midchain, is_mcast);
2147 }
2148
2149 VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm,
2150                                  vlib_node_runtime_t * node,
2151                                  vlib_frame_t * frame)
2152 {
2153   if (adj_are_counters_enabled ())
2154     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2155   else
2156     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2157 }
2158
2159 VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm,
2160                                        vlib_node_runtime_t * node,
2161                                        vlib_frame_t * frame)
2162 {
2163   if (adj_are_counters_enabled ())
2164     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2165   else
2166     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2167 }
2168
2169 VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm,
2170                                        vlib_node_runtime_t * node,
2171                                        vlib_frame_t * frame)
2172 {
2173   if (adj_are_counters_enabled ())
2174     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2175   else
2176     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2177 }
2178
2179 VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm,
2180                                   vlib_node_runtime_t * node,
2181                                   vlib_frame_t * frame)
2182 {
2183   if (adj_are_counters_enabled ())
2184     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2185   else
2186     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2187 }
2188
2189 VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
2190                                         vlib_node_runtime_t * node,
2191                                         vlib_frame_t * frame)
2192 {
2193   if (adj_are_counters_enabled ())
2194     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2195   else
2196     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2197 }
2198
2199 /* *INDENT-OFF* */
2200 VLIB_REGISTER_NODE (ip6_midchain_node) =
2201 {
2202   .name = "ip6-midchain",
2203   .vector_size = sizeof (u32),
2204   .format_trace = format_ip6_forward_next_trace,
2205   .sibling_of = "ip6-rewrite",
2206   };
2207
2208 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2209 {
2210   .name = "ip6-rewrite",
2211   .vector_size = sizeof (u32),
2212   .format_trace = format_ip6_rewrite_trace,
2213   .n_next_nodes = IP6_REWRITE_N_NEXT,
2214   .next_nodes =
2215   {
2216     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2217     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2218     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2219   },
2220 };
2221
2222 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2223   .name = "ip6-rewrite-bcast",
2224   .vector_size = sizeof (u32),
2225
2226   .format_trace = format_ip6_rewrite_trace,
2227   .sibling_of = "ip6-rewrite",
2228 };
2229
2230 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2231 {
2232   .name = "ip6-rewrite-mcast",
2233   .vector_size = sizeof (u32),
2234   .format_trace = format_ip6_rewrite_trace,
2235   .sibling_of = "ip6-rewrite",
2236 };
2237
2238
2239 VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
2240 {
2241   .name = "ip6-mcast-midchain",
2242   .vector_size = sizeof (u32),
2243   .format_trace = format_ip6_rewrite_trace,
2244   .sibling_of = "ip6-rewrite",
2245 };
2246
2247 /* *INDENT-ON* */
2248
2249 /*
2250  * Hop-by-Hop handling
2251  */
2252 #ifndef CLIB_MARCH_VARIANT
2253 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2254 #endif /* CLIB_MARCH_VARIANT */
2255
2256 #define foreach_ip6_hop_by_hop_error \
2257 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2258 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2259 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2260
2261 /* *INDENT-OFF* */
2262 typedef enum
2263 {
2264 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2265   foreach_ip6_hop_by_hop_error
2266 #undef _
2267   IP6_HOP_BY_HOP_N_ERROR,
2268 } ip6_hop_by_hop_error_t;
2269 /* *INDENT-ON* */
2270
2271 /*
2272  * Primary h-b-h handler trace support
2273  * We work pretty hard on the problem for obvious reasons
2274  */
2275 typedef struct
2276 {
2277   u32 next_index;
2278   u32 trace_len;
2279   u8 option_data[256];
2280 } ip6_hop_by_hop_trace_t;
2281
2282 extern vlib_node_registration_t ip6_hop_by_hop_node;
2283
2284 static char *ip6_hop_by_hop_error_strings[] = {
2285 #define _(sym,string) string,
2286   foreach_ip6_hop_by_hop_error
2287 #undef _
2288 };
2289
2290 #ifndef CLIB_MARCH_VARIANT
2291 u8 *
2292 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2293 {
2294   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2295   int total_len = va_arg (*args, int);
2296   ip6_hop_by_hop_option_t *opt0, *limit0;
2297   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2298   u8 type0;
2299   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2300               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2301
2302   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2303   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2304
2305   while (opt0 < limit0)
2306     {
2307       type0 = opt0->type;
2308       switch (type0)
2309         {
2310         case 0:         /* Pad, just stop */
2311           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2312           break;
2313
2314         default:
2315           if (hm->trace[type0])
2316             {
2317               s = (*hm->trace[type0]) (s, opt0);
2318             }
2319           else
2320             {
2321               s = format (s, "\n    unrecognized option %d length %d", type0,
2322                           opt0->length);
2323             }
2324           opt0 =
2325             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2326                                          sizeof (ip6_hop_by_hop_option_t));
2327           break;
2328         }
2329     }
2330   return s;
2331 }
2332 #endif
2333
2334 static u8 *
2335 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2336 {
2337   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2338   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2339   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2340   ip6_hop_by_hop_header_t *hbh0;
2341   ip6_hop_by_hop_option_t *opt0, *limit0;
2342   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2343
2344   u8 type0;
2345
2346   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2347
2348   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2349               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2350
2351   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2352   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2353
2354   while (opt0 < limit0)
2355     {
2356       type0 = opt0->type;
2357       switch (type0)
2358         {
2359         case 0:         /* Pad, just stop */
2360           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2361           break;
2362
2363         default:
2364           if (hm->trace[type0])
2365             {
2366               s = (*hm->trace[type0]) (s, opt0);
2367             }
2368           else
2369             {
2370               s = format (s, "\n    unrecognized option %d length %d", type0,
2371                           opt0->length);
2372             }
2373           opt0 =
2374             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2375                                          sizeof (ip6_hop_by_hop_option_t));
2376           break;
2377         }
2378     }
2379   return s;
2380 }
2381
2382 always_inline u8
2383 ip6_scan_hbh_options (vlib_buffer_t * b0,
2384                       ip6_header_t * ip0,
2385                       ip6_hop_by_hop_header_t * hbh0,
2386                       ip6_hop_by_hop_option_t * opt0,
2387                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2388 {
2389   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2390   u8 type0;
2391   u8 error0 = 0;
2392
2393   while (opt0 < limit0)
2394     {
2395       type0 = opt0->type;
2396       switch (type0)
2397         {
2398         case 0:         /* Pad1 */
2399           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2400           continue;
2401         case 1:         /* PadN */
2402           break;
2403         default:
2404           if (hm->options[type0])
2405             {
2406               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2407                 {
2408                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2409                   return (error0);
2410                 }
2411             }
2412           else
2413             {
2414               /* Unrecognized mandatory option, check the two high order bits */
2415               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2416                 {
2417                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2418                   break;
2419                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2420                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2421                   *next0 = IP_LOOKUP_NEXT_DROP;
2422                   break;
2423                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2424                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2425                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2426                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2427                                                ICMP6_parameter_problem_unrecognized_option,
2428                                                (u8 *) opt0 - (u8 *) ip0);
2429                   break;
2430                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2431                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2432                   if (!ip6_address_is_multicast (&ip0->dst_address))
2433                     {
2434                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2435                       icmp6_error_set_vnet_buffer (b0,
2436                                                    ICMP6_parameter_problem,
2437                                                    ICMP6_parameter_problem_unrecognized_option,
2438                                                    (u8 *) opt0 - (u8 *) ip0);
2439                     }
2440                   else
2441                     {
2442                       *next0 = IP_LOOKUP_NEXT_DROP;
2443                     }
2444                   break;
2445                 }
2446               return (error0);
2447             }
2448         }
2449       opt0 =
2450         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2451                                      sizeof (ip6_hop_by_hop_option_t));
2452     }
2453   return (error0);
2454 }
2455
2456 /*
2457  * Process the Hop-by-Hop Options header
2458  */
2459 VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
2460                                     vlib_node_runtime_t * node,
2461                                     vlib_frame_t * frame)
2462 {
2463   vlib_node_runtime_t *error_node =
2464     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2465   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2466   u32 n_left_from, *from, *to_next;
2467   ip_lookup_next_t next_index;
2468
2469   from = vlib_frame_vector_args (frame);
2470   n_left_from = frame->n_vectors;
2471   next_index = node->cached_next_index;
2472
2473   while (n_left_from > 0)
2474     {
2475       u32 n_left_to_next;
2476
2477       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2478
2479       while (n_left_from >= 4 && n_left_to_next >= 2)
2480         {
2481           u32 bi0, bi1;
2482           vlib_buffer_t *b0, *b1;
2483           u32 next0, next1;
2484           ip6_header_t *ip0, *ip1;
2485           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2486           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2487           u8 error0 = 0, error1 = 0;
2488
2489           /* Prefetch next iteration. */
2490           {
2491             vlib_buffer_t *p2, *p3;
2492
2493             p2 = vlib_get_buffer (vm, from[2]);
2494             p3 = vlib_get_buffer (vm, from[3]);
2495
2496             vlib_prefetch_buffer_header (p2, LOAD);
2497             vlib_prefetch_buffer_header (p3, LOAD);
2498
2499             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2500             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2501           }
2502
2503           /* Speculatively enqueue b0, b1 to the current next frame */
2504           to_next[0] = bi0 = from[0];
2505           to_next[1] = bi1 = from[1];
2506           from += 2;
2507           to_next += 2;
2508           n_left_from -= 2;
2509           n_left_to_next -= 2;
2510
2511           b0 = vlib_get_buffer (vm, bi0);
2512           b1 = vlib_get_buffer (vm, bi1);
2513
2514           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2515           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2516           ip_adjacency_t *adj0 = adj_get (adj_index0);
2517           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2518           ip_adjacency_t *adj1 = adj_get (adj_index1);
2519
2520           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2521           next0 = adj0->lookup_next_index;
2522           next1 = adj1->lookup_next_index;
2523
2524           ip0 = vlib_buffer_get_current (b0);
2525           ip1 = vlib_buffer_get_current (b1);
2526           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2527           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2528           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2529           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2530           limit0 =
2531             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2532                                          ((hbh0->length + 1) << 3));
2533           limit1 =
2534             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2535                                          ((hbh1->length + 1) << 3));
2536
2537           /*
2538            * Basic validity checks
2539            */
2540           if ((hbh0->length + 1) << 3 >
2541               clib_net_to_host_u16 (ip0->payload_length))
2542             {
2543               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2544               next0 = IP_LOOKUP_NEXT_DROP;
2545               goto outdual;
2546             }
2547           /* Scan the set of h-b-h options, process ones that we understand */
2548           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2549
2550           if ((hbh1->length + 1) << 3 >
2551               clib_net_to_host_u16 (ip1->payload_length))
2552             {
2553               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2554               next1 = IP_LOOKUP_NEXT_DROP;
2555               goto outdual;
2556             }
2557           /* Scan the set of h-b-h options, process ones that we understand */
2558           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2559
2560         outdual:
2561           /* Has the classifier flagged this buffer for special treatment? */
2562           if (PREDICT_FALSE
2563               ((error0 == 0)
2564                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2565             next0 = hm->next_override;
2566
2567           /* Has the classifier flagged this buffer for special treatment? */
2568           if (PREDICT_FALSE
2569               ((error1 == 0)
2570                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2571             next1 = hm->next_override;
2572
2573           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2574             {
2575               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2576                 {
2577                   ip6_hop_by_hop_trace_t *t =
2578                     vlib_add_trace (vm, node, b0, sizeof (*t));
2579                   u32 trace_len = (hbh0->length + 1) << 3;
2580                   t->next_index = next0;
2581                   /* Capture the h-b-h option verbatim */
2582                   trace_len =
2583                     trace_len <
2584                     ARRAY_LEN (t->option_data) ? trace_len :
2585                     ARRAY_LEN (t->option_data);
2586                   t->trace_len = trace_len;
2587                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2588                 }
2589               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2590                 {
2591                   ip6_hop_by_hop_trace_t *t =
2592                     vlib_add_trace (vm, node, b1, sizeof (*t));
2593                   u32 trace_len = (hbh1->length + 1) << 3;
2594                   t->next_index = next1;
2595                   /* Capture the h-b-h option verbatim */
2596                   trace_len =
2597                     trace_len <
2598                     ARRAY_LEN (t->option_data) ? trace_len :
2599                     ARRAY_LEN (t->option_data);
2600                   t->trace_len = trace_len;
2601                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2602                 }
2603
2604             }
2605
2606           b0->error = error_node->errors[error0];
2607           b1->error = error_node->errors[error1];
2608
2609           /* verify speculative enqueue, maybe switch current next frame */
2610           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2611                                            n_left_to_next, bi0, bi1, next0,
2612                                            next1);
2613         }
2614
2615       while (n_left_from > 0 && n_left_to_next > 0)
2616         {
2617           u32 bi0;
2618           vlib_buffer_t *b0;
2619           u32 next0;
2620           ip6_header_t *ip0;
2621           ip6_hop_by_hop_header_t *hbh0;
2622           ip6_hop_by_hop_option_t *opt0, *limit0;
2623           u8 error0 = 0;
2624
2625           /* Speculatively enqueue b0 to the current next frame */
2626           bi0 = from[0];
2627           to_next[0] = bi0;
2628           from += 1;
2629           to_next += 1;
2630           n_left_from -= 1;
2631           n_left_to_next -= 1;
2632
2633           b0 = vlib_get_buffer (vm, bi0);
2634           /*
2635            * Default use the next_index from the adjacency.
2636            * A HBH option rarely redirects to a different node
2637            */
2638           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2639           ip_adjacency_t *adj0 = adj_get (adj_index0);
2640           next0 = adj0->lookup_next_index;
2641
2642           ip0 = vlib_buffer_get_current (b0);
2643           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2644           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2645           limit0 =
2646             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2647                                          ((hbh0->length + 1) << 3));
2648
2649           /*
2650            * Basic validity checks
2651            */
2652           if ((hbh0->length + 1) << 3 >
2653               clib_net_to_host_u16 (ip0->payload_length))
2654             {
2655               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2656               next0 = IP_LOOKUP_NEXT_DROP;
2657               goto out0;
2658             }
2659
2660           /* Scan the set of h-b-h options, process ones that we understand */
2661           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2662
2663         out0:
2664           /* Has the classifier flagged this buffer for special treatment? */
2665           if (PREDICT_FALSE
2666               ((error0 == 0)
2667                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2668             next0 = hm->next_override;
2669
2670           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2671             {
2672               ip6_hop_by_hop_trace_t *t =
2673                 vlib_add_trace (vm, node, b0, sizeof (*t));
2674               u32 trace_len = (hbh0->length + 1) << 3;
2675               t->next_index = next0;
2676               /* Capture the h-b-h option verbatim */
2677               trace_len =
2678                 trace_len <
2679                 ARRAY_LEN (t->option_data) ? trace_len :
2680                 ARRAY_LEN (t->option_data);
2681               t->trace_len = trace_len;
2682               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2683             }
2684
2685           b0->error = error_node->errors[error0];
2686
2687           /* verify speculative enqueue, maybe switch current next frame */
2688           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2689                                            n_left_to_next, bi0, next0);
2690         }
2691       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2692     }
2693   return frame->n_vectors;
2694 }
2695
2696 /* *INDENT-OFF* */
2697 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2698 {
2699   .name = "ip6-hop-by-hop",
2700   .sibling_of = "ip6-lookup",
2701   .vector_size = sizeof (u32),
2702   .format_trace = format_ip6_hop_by_hop_trace,
2703   .type = VLIB_NODE_TYPE_INTERNAL,
2704   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2705   .error_strings = ip6_hop_by_hop_error_strings,
2706   .n_next_nodes = 0,
2707 };
2708 /* *INDENT-ON* */
2709
2710 static clib_error_t *
2711 ip6_hop_by_hop_init (vlib_main_t * vm)
2712 {
2713   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2714   clib_memset (hm->options, 0, sizeof (hm->options));
2715   clib_memset (hm->trace, 0, sizeof (hm->trace));
2716   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2717   return (0);
2718 }
2719
2720 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2721
2722 #ifndef CLIB_MARCH_VARIANT
2723 void
2724 ip6_hbh_set_next_override (uword next)
2725 {
2726   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2727
2728   hm->next_override = next;
2729 }
2730
2731 int
2732 ip6_hbh_register_option (u8 option,
2733                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2734                                       ip6_hop_by_hop_option_t * opt),
2735                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2736 {
2737   ip6_main_t *im = &ip6_main;
2738   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2739
2740   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2741
2742   /* Already registered */
2743   if (hm->options[option])
2744     return (-1);
2745
2746   hm->options[option] = options;
2747   hm->trace[option] = trace;
2748
2749   /* Set global variable */
2750   im->hbh_enabled = 1;
2751
2752   return (0);
2753 }
2754
2755 int
2756 ip6_hbh_unregister_option (u8 option)
2757 {
2758   ip6_main_t *im = &ip6_main;
2759   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2760
2761   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2762
2763   /* Not registered */
2764   if (!hm->options[option])
2765     return (-1);
2766
2767   hm->options[option] = NULL;
2768   hm->trace[option] = NULL;
2769
2770   /* Disable global knob if this was the last option configured */
2771   int i;
2772   bool found = false;
2773   for (i = 0; i < 256; i++)
2774     {
2775       if (hm->options[option])
2776         {
2777           found = true;
2778           break;
2779         }
2780     }
2781   if (!found)
2782     im->hbh_enabled = 0;
2783
2784   return (0);
2785 }
2786
2787 /* Global IP6 main. */
2788 ip6_main_t ip6_main;
2789 #endif
2790
2791 static clib_error_t *
2792 ip6_lookup_init (vlib_main_t * vm)
2793 {
2794   ip6_main_t *im = &ip6_main;
2795   clib_error_t *error;
2796   uword i;
2797
2798   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2799     return error;
2800
2801   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2802     {
2803       u32 j, i0, i1;
2804
2805       i0 = i / 32;
2806       i1 = i % 32;
2807
2808       for (j = 0; j < i0; j++)
2809         im->fib_masks[i].as_u32[j] = ~0;
2810
2811       if (i1)
2812         im->fib_masks[i].as_u32[i0] =
2813           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2814     }
2815
2816   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2817
2818   /* Create FIB with index 0 and table id of 0. */
2819   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2820                                      FIB_SOURCE_DEFAULT_ROUTE);
2821   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2822                                       MFIB_SOURCE_DEFAULT_ROUTE);
2823
2824   {
2825     pg_node_t *pn;
2826     pn = pg_get_node (ip6_lookup_node.index);
2827     pn->unformat_edit = unformat_pg_ip6_header;
2828   }
2829
2830   /* Unless explicitly configured, don't process HBH options */
2831   im->hbh_enabled = 0;
2832
2833   return error;
2834 }
2835
2836 VLIB_INIT_FUNCTION (ip6_lookup_init);
2837
2838 static clib_error_t *
2839 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2840                               unformat_input_t * input,
2841                               vlib_cli_command_t * cmd)
2842 {
2843   int matched = 0;
2844   u32 table_id = 0;
2845   u32 flow_hash_config = 0;
2846   int rv;
2847
2848   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2849     {
2850       if (unformat (input, "table %d", &table_id))
2851         matched = 1;
2852 #define _(a, b, v)                                                            \
2853   else if (unformat (input, #a))                                              \
2854   {                                                                           \
2855     flow_hash_config |= v;                                                    \
2856     matched = 1;                                                              \
2857   }
2858       foreach_flow_hash_bit
2859 #undef _
2860         else
2861         break;
2862     }
2863
2864   if (matched == 0)
2865     return clib_error_return (0, "unknown input `%U'",
2866                               format_unformat_error, input);
2867
2868   rv = ip_flow_hash_set (AF_IP6, table_id, flow_hash_config);
2869   switch (rv)
2870     {
2871     case 0:
2872       break;
2873
2874     case -1:
2875       return clib_error_return (0, "no such FIB table %d", table_id);
2876
2877     default:
2878       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2879       break;
2880     }
2881
2882   return 0;
2883 }
2884
2885 /*?
2886  * Configure the set of IPv6 fields used by the flow hash.
2887  *
2888  * @cliexpar
2889  * @parblock
2890  * Example of how to set the flow hash on a given table:
2891  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2892  *
2893  * Example of display the configured flow hash:
2894  * @cliexstart{show ip6 fib}
2895  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2896  * @::/0
2897  *   unicast-ip6-chain
2898  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2899  *     [0] [@0]: dpo-drop ip6
2900  * fe80::/10
2901  *   unicast-ip6-chain
2902  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2903  *     [0] [@2]: dpo-receive
2904  * ff02::1/128
2905  *   unicast-ip6-chain
2906  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2907  *     [0] [@2]: dpo-receive
2908  * ff02::2/128
2909  *   unicast-ip6-chain
2910  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2911  *     [0] [@2]: dpo-receive
2912  * ff02::16/128
2913  *   unicast-ip6-chain
2914  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2915  *     [0] [@2]: dpo-receive
2916  * ff02::1:ff00:0/104
2917  *   unicast-ip6-chain
2918  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2919  *     [0] [@2]: dpo-receive
2920  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2921  * @::/0
2922  *   unicast-ip6-chain
2923  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2924  *     [0] [@0]: dpo-drop ip6
2925  * @::a:1:1:0:4/126
2926  *   unicast-ip6-chain
2927  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2928  *     [0] [@4]: ipv6-glean: af_packet0
2929  * @::a:1:1:0:7/128
2930  *   unicast-ip6-chain
2931  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2932  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2933  * fe80::/10
2934  *   unicast-ip6-chain
2935  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2936  *     [0] [@2]: dpo-receive
2937  * fe80::fe:3eff:fe3e:9222/128
2938  *   unicast-ip6-chain
2939  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2940  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2941  * ff02::1/128
2942  *   unicast-ip6-chain
2943  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2944  *     [0] [@2]: dpo-receive
2945  * ff02::2/128
2946  *   unicast-ip6-chain
2947  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2948  *     [0] [@2]: dpo-receive
2949  * ff02::16/128
2950  *   unicast-ip6-chain
2951  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2952  *     [0] [@2]: dpo-receive
2953  * ff02::1:ff00:0/104
2954  *   unicast-ip6-chain
2955  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2956  *     [0] [@2]: dpo-receive
2957  * @cliexend
2958  * @endparblock
2959 ?*/
2960 /* *INDENT-OFF* */
2961 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
2962   .path = "set ip6 flow-hash",
2963   .short_help = "set ip6 flow-hash table <table-id> [src] [dst] [sport] "
2964                 "[dport] [proto] [reverse] [flowlabel]",
2965   .function = set_ip6_flow_hash_command_fn,
2966 };
2967 /* *INDENT-ON* */
2968
2969 static clib_error_t *
2970 show_ip6_local_command_fn (vlib_main_t * vm,
2971                            unformat_input_t * input, vlib_cli_command_t * cmd)
2972 {
2973   ip6_main_t *im = &ip6_main;
2974   ip_lookup_main_t *lm = &im->lookup_main;
2975   int i;
2976
2977   vlib_cli_output (vm, "Protocols handled by ip6_local");
2978   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
2979     {
2980       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2981         {
2982
2983           u32 node_index = vlib_get_node (vm,
2984                                           ip6_local_node.index)->
2985             next_nodes[lm->local_next_by_ip_protocol[i]];
2986           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
2987                            node_index);
2988         }
2989     }
2990   return 0;
2991 }
2992
2993
2994
2995 /*?
2996  * Display the set of protocols handled by the local IPv6 stack.
2997  *
2998  * @cliexpar
2999  * Example of how to display local protocol table:
3000  * @cliexstart{show ip6 local}
3001  * Protocols handled by ip6_local
3002  * 17
3003  * 43
3004  * 58
3005  * 115
3006  * @cliexend
3007 ?*/
3008 /* *INDENT-OFF* */
3009 VLIB_CLI_COMMAND (show_ip6_local, static) =
3010 {
3011   .path = "show ip6 local",
3012   .function = show_ip6_local_command_fn,
3013   .short_help = "show ip6 local",
3014 };
3015 /* *INDENT-ON* */
3016
3017 #ifndef CLIB_MARCH_VARIANT
3018 int
3019 vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
3020 {
3021   vnet_main_t *vnm = vnet_get_main ();
3022   vnet_interface_main_t *im = &vnm->interface_main;
3023   ip6_main_t *ipm = &ip6_main;
3024   ip_lookup_main_t *lm = &ipm->lookup_main;
3025   vnet_classify_main_t *cm = &vnet_classify_main;
3026   ip6_address_t *if_addr;
3027
3028   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3029     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3030
3031   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3032     return VNET_API_ERROR_NO_SUCH_ENTRY;
3033
3034   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3035   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3036
3037   if_addr = ip6_interface_first_address (ipm, sw_if_index);
3038
3039   if (NULL != if_addr)
3040     {
3041       fib_prefix_t pfx = {
3042         .fp_len = 128,
3043         .fp_proto = FIB_PROTOCOL_IP6,
3044         .fp_addr.ip6 = *if_addr,
3045       };
3046       u32 fib_index;
3047
3048       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3049                                                        sw_if_index);
3050       if (table_index != (u32) ~ 0)
3051         {
3052           dpo_id_t dpo = DPO_INVALID;
3053           dpo_set (&dpo,
3054                    DPO_CLASSIFY,
3055                    DPO_PROTO_IP6,
3056                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3057           fib_table_entry_special_dpo_add (fib_index,
3058                                            &pfx,
3059                                            FIB_SOURCE_CLASSIFY,
3060                                            FIB_ENTRY_FLAG_NONE, &dpo);
3061           dpo_reset (&dpo);
3062         }
3063       else
3064         {
3065           fib_table_entry_special_remove (fib_index,
3066                                           &pfx, FIB_SOURCE_CLASSIFY);
3067         }
3068     }
3069
3070   return 0;
3071 }
3072 #endif
3073
3074 static clib_error_t *
3075 set_ip6_classify_command_fn (vlib_main_t * vm,
3076                              unformat_input_t * input,
3077                              vlib_cli_command_t * cmd)
3078 {
3079   u32 table_index = ~0;
3080   int table_index_set = 0;
3081   u32 sw_if_index = ~0;
3082   int rv;
3083
3084   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3085     {
3086       if (unformat (input, "table-index %d", &table_index))
3087         table_index_set = 1;
3088       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3089                          vnet_get_main (), &sw_if_index))
3090         ;
3091       else
3092         break;
3093     }
3094
3095   if (table_index_set == 0)
3096     return clib_error_return (0, "classify table-index must be specified");
3097
3098   if (sw_if_index == ~0)
3099     return clib_error_return (0, "interface / subif must be specified");
3100
3101   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3102
3103   switch (rv)
3104     {
3105     case 0:
3106       break;
3107
3108     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3109       return clib_error_return (0, "No such interface");
3110
3111     case VNET_API_ERROR_NO_SUCH_ENTRY:
3112       return clib_error_return (0, "No such classifier table");
3113     }
3114   return 0;
3115 }
3116
3117 /*?
3118  * Assign a classification table to an interface. The classification
3119  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3120  * commands. Once the table is create, use this command to filter packets
3121  * on an interface.
3122  *
3123  * @cliexpar
3124  * Example of how to assign a classification table to an interface:
3125  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3126 ?*/
3127 /* *INDENT-OFF* */
3128 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3129 {
3130   .path = "set ip6 classify",
3131   .short_help =
3132   "set ip6 classify intfc <interface> table-index <classify-idx>",
3133   .function = set_ip6_classify_command_fn,
3134 };
3135 /* *INDENT-ON* */
3136
3137 /*
3138  * fd.io coding-style-patch-verification: ON
3139  *
3140  * Local Variables:
3141  * eval: (c-set-style "gnu")
3142  * End:
3143  */