src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip_frag.h>
  43 #include <vnet/ip/ip6_link.h>
  44 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  45 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  46 #include <vppinfra/cache.h>
  47 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  48 #include <vnet/fib/ip6_fib.h>
  49 #include <vnet/mfib/ip6_mfib.h>
  50 #include <vnet/dpo/load_balance_map.h>
  51 #include <vnet/dpo/classify_dpo.h>
  52 #include <vnet/classify/vnet_classify.h>
  53
  54 #ifndef CLIB_MARCH_VARIANT
  55 #include <vppinfra/bihash_template.c>
  56 #endif
  57 #include <vnet/ip/ip6_forward.h>
  58 #include <vnet/interface_output.h>
  59
  60 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  61 #define OI_DECAP   0x80000000
  62
  63 static void
  64 ip6_add_interface_prefix_routes (ip6_main_t * im,
  65                                  u32 sw_if_index,
  66                                  u32 fib_index,
  67                                  ip6_address_t * address, u32 address_length)
  68 {
  69   ip_lookup_main_t *lm = &im->lookup_main;
  70   ip_interface_prefix_t *if_prefix;
  71
  72   /* *INDENT-OFF* */
  73   ip_interface_prefix_key_t key = {
  74     .prefix = {
  75       .fp_len = address_length,
  76       .fp_proto = FIB_PROTOCOL_IP6,
  77       .fp_addr.ip6 = {
  78         .as_u64 = {
  79           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
  80           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
  81         },
  82       },
  83     },
  84     .sw_if_index = sw_if_index,
  85   };
  86   /* *INDENT-ON* */
  87
  88   /* If prefix already set on interface, just increment ref count & return */
  89   if_prefix = ip_get_interface_prefix (lm, &key);
  90   if (if_prefix)
  91     {
  92       if_prefix->ref_count += 1;
  93       return;
  94     }
  95
  96   /* New prefix - allocate a pool entry, initialize it, add to the hash */
  97   pool_get (lm->if_prefix_pool, if_prefix);
  98   if_prefix->ref_count = 1;
  99   clib_memcpy (&if_prefix->key, &key, sizeof (key));
 100   mhash_set (&lm->prefix_to_if_prefix_index, &key,
 101              if_prefix - lm->if_prefix_pool, 0 /* old value */ );
 102
 103   /* length < 128 - add glean */
 104   if (address_length < 128)
 105     {
 106       /* set the glean route for the prefix */
 107       fib_table_entry_update_one_path (fib_index, &key.prefix,
 108                                        FIB_SOURCE_INTERFACE,
 109                                        (FIB_ENTRY_FLAG_CONNECTED |
 110                                         FIB_ENTRY_FLAG_ATTACHED),
 111                                        DPO_PROTO_IP6,
 112                                        /* No next-hop address */
 113                                        NULL, sw_if_index,
 114                                        /* invalid FIB index */
 115                                        ~0, 1,
 116                                        /* no out-label stack */
 117                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
 118     }
 119 }
 120
 121 static void
 122 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 123                           ip6_main_t * im, u32 fib_index,
 124                           ip_interface_address_t * a)
 125 {
 126   ip_lookup_main_t *lm = &im->lookup_main;
 127   ip6_address_t *address = ip_interface_address_get_address (lm, a);
 128   fib_prefix_t pfx = {
 129     .fp_len = a->address_length,
 130     .fp_proto = FIB_PROTOCOL_IP6,
 131     .fp_addr.ip6 = *address,
 132   };
 133
 134   /* set special routes for the prefix if needed */
 135   ip6_add_interface_prefix_routes (im, sw_if_index, fib_index,
 136                                    address, a->address_length);
 137
 138   pfx.fp_len = 128;
 139   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 140     {
 141       u32 classify_table_index =
 142         lm->classify_table_index_by_sw_if_index[sw_if_index];
 143       if (classify_table_index != (u32) ~ 0)
 144         {
 145           dpo_id_t dpo = DPO_INVALID;
 146
 147           dpo_set (&dpo,
 148                    DPO_CLASSIFY,
 149                    DPO_PROTO_IP6,
 150                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 151
 152           fib_table_entry_special_dpo_add (fib_index,
 153                                            &pfx,
 154                                            FIB_SOURCE_CLASSIFY,
 155                                            FIB_ENTRY_FLAG_NONE, &dpo);
 156           dpo_reset (&dpo);
 157         }
 158     }
 159
 160   fib_table_entry_update_one_path (fib_index, &pfx,
 161                                    FIB_SOURCE_INTERFACE,
 162                                    (FIB_ENTRY_FLAG_CONNECTED |
 163                                     FIB_ENTRY_FLAG_LOCAL),
 164                                    DPO_PROTO_IP6,
 165                                    &pfx.fp_addr,
 166                                    sw_if_index, ~0,
 167                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 168 }
 169
 170 static void
 171 ip6_del_interface_prefix_routes (ip6_main_t * im,
 172                                  u32 sw_if_index,
 173                                  u32 fib_index,
 174                                  ip6_address_t * address, u32 address_length)
 175 {
 176   ip_lookup_main_t *lm = &im->lookup_main;
 177   ip_interface_prefix_t *if_prefix;
 178
 179   /* *INDENT-OFF* */
 180   ip_interface_prefix_key_t key = {
 181     .prefix = {
 182       .fp_len = address_length,
 183       .fp_proto = FIB_PROTOCOL_IP6,
 184       .fp_addr.ip6 = {
 185         .as_u64 = {
 186           address->as_u64[0] & im->fib_masks[address_length].as_u64[0],
 187           address->as_u64[1] & im->fib_masks[address_length].as_u64[1],
 188         },
 189       },
 190     },
 191     .sw_if_index = sw_if_index,
 192   };
 193   /* *INDENT-ON* */
 194
 195   if_prefix = ip_get_interface_prefix (lm, &key);
 196   if (!if_prefix)
 197     {
 198       clib_warning ("Prefix not found while deleting %U",
 199                     format_ip4_address_and_length, address, address_length);
 200       return;
 201     }
 202
 203   /* If not deleting last intf addr in prefix, decrement ref count & return */
 204   if_prefix->ref_count -= 1;
 205   if (if_prefix->ref_count > 0)
 206     return;
 207
 208   /* length <= 128, delete glean route */
 209   if (address_length <= 128)
 210     {
 211       /* remove glean route for prefix */
 212       fib_table_entry_delete (fib_index, &key.prefix, FIB_SOURCE_INTERFACE);
 213     }
 214
 215   mhash_unset (&lm->prefix_to_if_prefix_index, &key, 0 /* old_value */ );
 216   pool_put (lm->if_prefix_pool, if_prefix);
 217 }
 218
 219 static void
 220 ip6_del_interface_routes (u32 sw_if_index, ip6_main_t * im,
 221                           u32 fib_index,
 222                           ip6_address_t * address, u32 address_length)
 223 {
 224   fib_prefix_t pfx = {
 225     .fp_len = 128,
 226     .fp_proto = FIB_PROTOCOL_IP6,
 227     .fp_addr.ip6 = *address,
 228   };
 229
 230   /* delete special routes for the prefix if needed */
 231   ip6_del_interface_prefix_routes (im, sw_if_index, fib_index,
 232                                    address, address_length);
 233
 234   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 235 }
 236
 237 #ifndef CLIB_MARCH_VARIANT
 238 void
 239 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 240 {
 241   ip6_main_t *im = &ip6_main;
 242   vnet_main_t *vnm = vnet_get_main ();
 243   vnet_hw_interface_t *hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
 244
 245   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 246
 247   /*
 248    * enable/disable only on the 1<->0 transition
 249    */
 250   if (is_enable)
 251     {
 252       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 253         return;
 254     }
 255   else
 256     {
 257       /* The ref count is 0 when an address is removed from an interface that has
 258        * no address - this is not a ciritical error */
 259       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 260           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 261         return;
 262     }
 263
 264   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 265                                !is_enable, 0, 0);
 266
 267   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 268                                sw_if_index, !is_enable, 0, 0);
 269
 270   if (is_enable)
 271     hi->l3_if_count++;
 272   else if (hi->l3_if_count)
 273     hi->l3_if_count--;
 274 }
 275
 276 /* get first interface address */
 277 ip6_address_t *
 278 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 279 {
 280   ip_lookup_main_t *lm = &im->lookup_main;
 281   ip_interface_address_t *ia = 0;
 282   ip6_address_t *result = 0;
 283
 284   /* *INDENT-OFF* */
 285   foreach_ip_interface_address (lm, ia, sw_if_index,
 286                                 1 /* honor unnumbered */,
 287   ({
 288     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 289     result = a;
 290     break;
 291   }));
 292   /* *INDENT-ON* */
 293   return result;
 294 }
 295
 296 clib_error_t *
 297 ip6_add_del_interface_address (vlib_main_t * vm,
 298                                u32 sw_if_index,
 299                                ip6_address_t * address,
 300                                u32 address_length, u32 is_del)
 301 {
 302   vnet_main_t *vnm = vnet_get_main ();
 303   ip6_main_t *im = &ip6_main;
 304   ip_lookup_main_t *lm = &im->lookup_main;
 305   clib_error_t *error = NULL;
 306   u32 if_address_index;
 307   ip6_address_fib_t ip6_af, *addr_fib = 0;
 308   const ip6_address_t *ll_addr;
 309
 310   /* local0 interface doesn't support IP addressing */
 311   if (sw_if_index == 0)
 312     {
 313       return
 314         clib_error_create ("local0 interface doesn't support IP addressing");
 315     }
 316
 317   if (ip6_address_is_link_local_unicast (address))
 318     {
 319       if (address_length != 128)
 320         {
 321           vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
 322           return
 323             clib_error_create
 324             ("prefix length of link-local address must be 128");
 325         }
 326       if (!is_del)
 327         {
 328           int rv;
 329
 330           rv = ip6_link_set_local_address (sw_if_index, address);
 331
 332           if (rv)
 333             {
 334               vnm->api_errno = rv;
 335               return clib_error_create ("address not assignable");
 336             }
 337         }
 338       else
 339         {
 340           ll_addr = ip6_get_link_local_address (sw_if_index);
 341           if (ip6_address_is_equal (ll_addr, address))
 342             {
 343               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_DELETABLE;
 344               return clib_error_create ("address not deletable");
 345             }
 346           else
 347             {
 348               vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 349               return clib_error_create ("address not found");
 350             }
 351         }
 352
 353       return (NULL);
 354     }
 355
 356   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 357   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 358
 359   ip6_addr_fib_init (&ip6_af, address,
 360                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 361   vec_add1 (addr_fib, ip6_af);
 362
 363   /* *INDENT-OFF* */
 364   if (!is_del)
 365     {
 366       /* When adding an address check that it does not conflict
 367          with an existing address on any interface in this table. */
 368       ip_interface_address_t *ia;
 369       vnet_sw_interface_t *sif;
 370
 371       pool_foreach(sif, vnm->interface_main.sw_interfaces,
 372       ({
 373           if (im->fib_index_by_sw_if_index[sw_if_index] ==
 374               im->fib_index_by_sw_if_index[sif->sw_if_index])
 375             {
 376               foreach_ip_interface_address
 377                 (&im->lookup_main, ia, sif->sw_if_index,
 378                  0 /* honor unnumbered */ ,
 379                  ({
 380                    ip6_address_t * x =
 381                      ip_interface_address_get_address
 382                      (&im->lookup_main, ia);
 383
 384                    if (ip6_destination_matches_route
 385                        (im, address, x, ia->address_length) ||
 386                        ip6_destination_matches_route (im,
 387                                                       x,
 388                                                       address,
 389                                                       address_length))
 390                      {
 391                        /* an intf may have >1 addr from the same prefix */
 392                        if ((sw_if_index == sif->sw_if_index) &&
 393                            (ia->address_length == address_length) &&
 394                            !ip6_address_is_equal (x, address))
 395                          continue;
 396
 397                        if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
 398                          /* if the address we're comparing against is stale
 399                           * then the CP has not added this one back yet, maybe
 400                           * it never will, so we have to assume it won't and
 401                           * ignore it. if it does add it back, then it will fail
 402                           * because this one is now present */
 403                          continue;
 404
 405                        /* error if the length or intf was different */
 406                        vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 407                        error =  clib_error_create
 408                          ("failed to add %U which conflicts with %U for interface %U",
 409                           format_ip6_address_and_length, address,
 410                           address_length,
 411                           format_ip6_address_and_length, x,
 412                           ia->address_length,
 413                           format_vnet_sw_if_index_name, vnm,
 414                           sif->sw_if_index);
 415                        goto done;
 416                      }
 417                  }));
 418             }
 419       }));
 420     }
 421   /* *INDENT-ON* */
 422
 423   if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
 424
 425   if (is_del)
 426     {
 427       if (~0 == if_address_index)
 428         {
 429           vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
 430           error = clib_error_create ("%U not found for interface %U",
 431                                      lm->format_address_and_length,
 432                                      addr_fib, address_length,
 433                                      format_vnet_sw_if_index_name, vnm,
 434                                      sw_if_index);
 435           goto done;
 436         }
 437
 438       error = ip_interface_address_del (lm, vnm, if_address_index, addr_fib,
 439                                         address_length, sw_if_index);
 440       if (error)
 441         goto done;
 442     }
 443   else
 444     {
 445       if (~0 != if_address_index)
 446         {
 447           ip_interface_address_t *ia;
 448
 449           ia = pool_elt_at_index (lm->if_address_pool, if_address_index);
 450
 451           if (ia->flags & IP_INTERFACE_ADDRESS_FLAG_STALE)
 452             {
 453               if (ia->sw_if_index == sw_if_index)
 454                 {
 455                   /* re-adding an address during the replace action.
 456                    * consdier this the update. clear the flag and
 457                    * we're done */
 458                   ia->flags &= ~IP_INTERFACE_ADDRESS_FLAG_STALE;
 459                   goto done;
 460                 }
 461               else
 462                 {
 463                   /* The prefix is moving from one interface to another.
 464                    * delete the stale and add the new */
 465                   ip6_add_del_interface_address (vm,
 466                                                  ia->sw_if_index,
 467                                                  address, address_length, 1);
 468                   ia = NULL;
 469                   error = ip_interface_address_add (lm, sw_if_index,
 470                                                     addr_fib, address_length,
 471                                                     &if_address_index);
 472                 }
 473             }
 474           else
 475             {
 476               vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
 477               error = clib_error_create
 478                 ("Prefix %U already found on interface %U",
 479                  lm->format_address_and_length, addr_fib, address_length,
 480                  format_vnet_sw_if_index_name, vnm, ia->sw_if_index);
 481             }
 482         }
 483       else
 484         error = ip_interface_address_add (lm, sw_if_index,
 485                                           addr_fib, address_length,
 486                                           &if_address_index);
 487     }
 488
 489   if (error)
 490     goto done;
 491
 492   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 493   if (!is_del)
 494     ip6_link_enable (sw_if_index, NULL);
 495
 496   /* intf addr routes are added/deleted on admin up/down */
 497   if (vnet_sw_interface_is_admin_up (vnm, sw_if_index))
 498     {
 499       if (is_del)
 500         ip6_del_interface_routes (sw_if_index,
 501                                   im, ip6_af.fib_index, address,
 502                                   address_length);
 503       else
 504         ip6_add_interface_routes (vnm, sw_if_index,
 505                                   im, ip6_af.fib_index,
 506                                   pool_elt_at_index (lm->if_address_pool,
 507                                                      if_address_index));
 508     }
 509
 510   ip6_add_del_interface_address_callback_t *cb;
 511   vec_foreach (cb, im->add_del_interface_address_callbacks)
 512     cb->function (im, cb->function_opaque, sw_if_index,
 513                   address, address_length, if_address_index, is_del);
 514
 515   if (is_del)
 516     ip6_link_disable (sw_if_index);
 517
 518 done:
 519   vec_free (addr_fib);
 520   return error;
 521 }
 522
 523 #endif
 524
 525 static clib_error_t *
 526 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 527 {
 528   ip6_main_t *im = &ip6_main;
 529   ip_interface_address_t *ia;
 530   ip6_address_t *a;
 531   u32 is_admin_up, fib_index;
 532
 533   /* Fill in lookup tables with default table (0). */
 534   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 535
 536   vec_validate_init_empty (im->
 537                            lookup_main.if_address_pool_index_by_sw_if_index,
 538                            sw_if_index, ~0);
 539
 540   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 541
 542   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 543
 544   /* *INDENT-OFF* */
 545   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 546                                 0 /* honor unnumbered */,
 547   ({
 548     a = ip_interface_address_get_address (&im->lookup_main, ia);
 549     if (is_admin_up)
 550       ip6_add_interface_routes (vnm, sw_if_index,
 551                                 im, fib_index,
 552                                 ia);
 553     else
 554       ip6_del_interface_routes (sw_if_index, im, fib_index,
 555                                 a, ia->address_length);
 556   }));
 557   /* *INDENT-ON* */
 558
 559   return 0;
 560 }
 561
 562 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 563
 564 /* Built-in ip6 unicast rx feature path definition */
 565 /* *INDENT-OFF* */
 566 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 567 {
 568   .arc_name  = "ip6-unicast",
 569   .start_nodes = VNET_FEATURES ("ip6-input"),
 570   .last_in_arc = "ip6-lookup",
 571   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 572 };
 573
 574 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 575 {
 576   .arc_name = "ip6-unicast",
 577   .node_name = "ip6-flow-classify",
 578   .runs_before = VNET_FEATURES ("ip6-inacl"),
 579 };
 580
 581 VNET_FEATURE_INIT (ip6_inacl, static) =
 582 {
 583   .arc_name = "ip6-unicast",
 584   .node_name = "ip6-inacl",
 585   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 586 };
 587
 588 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 589 {
 590   .arc_name = "ip6-unicast",
 591   .node_name = "ip6-policer-classify",
 592   .runs_before = VNET_FEATURES ("ipsec6-input-feature"),
 593 };
 594
 595 VNET_FEATURE_INIT (ip6_ipsec, static) =
 596 {
 597   .arc_name = "ip6-unicast",
 598   .node_name = "ipsec6-input-feature",
 599   .runs_before = VNET_FEATURES ("l2tp-decap"),
 600 };
 601
 602 VNET_FEATURE_INIT (ip6_l2tp, static) =
 603 {
 604   .arc_name = "ip6-unicast",
 605   .node_name = "l2tp-decap",
 606   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 607 };
 608
 609 VNET_FEATURE_INIT (ip6_vpath, static) =
 610 {
 611   .arc_name = "ip6-unicast",
 612   .node_name = "vpath-input-ip6",
 613   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 614 };
 615
 616 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 617 {
 618   .arc_name = "ip6-unicast",
 619   .node_name = "ip6-vxlan-bypass",
 620   .runs_before = VNET_FEATURES ("ip6-lookup"),
 621 };
 622
 623 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 624 {
 625   .arc_name = "ip6-unicast",
 626   .node_name = "ip6-not-enabled",
 627   .runs_before = VNET_FEATURES ("ip6-lookup"),
 628 };
 629
 630 VNET_FEATURE_INIT (ip6_lookup, static) =
 631 {
 632   .arc_name = "ip6-unicast",
 633   .node_name = "ip6-lookup",
 634   .runs_before = 0,  /*last feature*/
 635 };
 636
 637 /* Built-in ip6 multicast rx feature path definition (none now) */
 638 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 639 {
 640   .arc_name  = "ip6-multicast",
 641   .start_nodes = VNET_FEATURES ("ip6-input"),
 642   .last_in_arc = "ip6-mfib-forward-lookup",
 643   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 644 };
 645
 646 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 647   .arc_name = "ip6-multicast",
 648   .node_name = "vpath-input-ip6",
 649   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 650 };
 651
 652 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 653   .arc_name = "ip6-multicast",
 654   .node_name = "ip6-not-enabled",
 655   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 656 };
 657
 658 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 659   .arc_name = "ip6-multicast",
 660   .node_name = "ip6-mfib-forward-lookup",
 661   .runs_before = 0, /* last feature */
 662 };
 663
 664 /* Built-in ip4 tx feature path definition */
 665 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 666 {
 667   .arc_name  = "ip6-output",
 668   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 669   .last_in_arc = "interface-output",
 670   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 671 };
 672
 673 VNET_FEATURE_INIT (ip6_outacl, static) = {
 674   .arc_name = "ip6-output",
 675   .node_name = "ip6-outacl",
 676   .runs_before = VNET_FEATURES ("ipsec6-output-feature"),
 677 };
 678
 679 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 680   .arc_name = "ip6-output",
 681   .node_name = "ipsec6-output-feature",
 682   .runs_before = VNET_FEATURES ("interface-output"),
 683 };
 684
 685 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 686   .arc_name = "ip6-output",
 687   .node_name = "interface-output",
 688   .runs_before = 0, /* not before any other features */
 689 };
 690 /* *INDENT-ON* */
 691
 692 static clib_error_t *
 693 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 694 {
 695   ip6_main_t *im = &ip6_main;
 696
 697   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 698   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 699
 700   if (!is_add)
 701     {
 702       /* Ensure that IPv6 is disabled */
 703       ip6_main_t *im6 = &ip6_main;
 704       ip_lookup_main_t *lm6 = &im6->lookup_main;
 705       ip_interface_address_t *ia = 0;
 706       ip6_address_t *address;
 707       vlib_main_t *vm = vlib_get_main ();
 708
 709       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
 710       /* *INDENT-OFF* */
 711       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
 712       ({
 713         address = ip_interface_address_get_address (lm6, ia);
 714         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 715       }));
 716       /* *INDENT-ON* */
 717       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 718     }
 719
 720   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 721                                is_add, 0, 0);
 722
 723   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 724                                sw_if_index, is_add, 0, 0);
 725
 726   return /* no error */ 0;
 727 }
 728
 729 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 730
 731 VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 732                                 vlib_node_runtime_t * node,
 733                                 vlib_frame_t * frame)
 734 {
 735   return ip6_lookup_inline (vm, node, frame);
 736 }
 737
 738 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 739
 740 /* *INDENT-OFF* */
 741 VLIB_REGISTER_NODE (ip6_lookup_node) =
 742 {
 743   .name = "ip6-lookup",
 744   .vector_size = sizeof (u32),
 745   .format_trace = format_ip6_lookup_trace,
 746   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 747   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 748 };
 749 /* *INDENT-ON* */
 750
 751 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
 752                                       vlib_node_runtime_t * node,
 753                                       vlib_frame_t * frame)
 754 {
 755   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 756   u32 n_left, *from;
 757   u32 thread_index = vm->thread_index;
 758   ip6_main_t *im = &ip6_main;
 759   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 760   u16 nexts[VLIB_FRAME_SIZE], *next;
 761
 762   from = vlib_frame_vector_args (frame);
 763   n_left = frame->n_vectors;
 764   next = nexts;
 765
 766   vlib_get_buffers (vm, from, bufs, n_left);
 767
 768   while (n_left >= 4)
 769     {
 770       const load_balance_t *lb0, *lb1;
 771       const ip6_header_t *ip0, *ip1;
 772       u32 lbi0, hc0, lbi1, hc1;
 773       const dpo_id_t *dpo0, *dpo1;
 774
 775       /* Prefetch next iteration. */
 776       {
 777         vlib_prefetch_buffer_header (b[2], STORE);
 778         vlib_prefetch_buffer_header (b[3], STORE);
 779
 780         CLIB_PREFETCH (b[2]->data, sizeof (ip0[0]), STORE);
 781         CLIB_PREFETCH (b[3]->data, sizeof (ip0[0]), STORE);
 782       }
 783
 784       ip0 = vlib_buffer_get_current (b[0]);
 785       ip1 = vlib_buffer_get_current (b[1]);
 786       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 787       lbi1 = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
 788
 789       lb0 = load_balance_get (lbi0);
 790       lb1 = load_balance_get (lbi1);
 791
 792       /*
 793        * this node is for via FIBs we can re-use the hash value from the
 794        * to node if present.
 795        * We don't want to use the same hash value at each level in the recursion
 796        * graph as that would lead to polarisation
 797        */
 798       hc0 = hc1 = 0;
 799
 800       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 801         {
 802           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 803             {
 804               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 805                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 806             }
 807           else
 808             {
 809               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 810                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 811             }
 812           dpo0 = load_balance_get_fwd_bucket
 813             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 814         }
 815       else
 816         {
 817           dpo0 = load_balance_get_bucket_i (lb0, 0);
 818         }
 819       if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 820         {
 821           if (PREDICT_TRUE (vnet_buffer (b[1])->ip.flow_hash))
 822             {
 823               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 824                 vnet_buffer (b[1])->ip.flow_hash >> 1;
 825             }
 826           else
 827             {
 828               hc1 = vnet_buffer (b[1])->ip.flow_hash =
 829                 ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 830             }
 831           dpo1 = load_balance_get_fwd_bucket
 832             (lb1, (hc1 & (lb1->lb_n_buckets_minus_1)));
 833         }
 834       else
 835         {
 836           dpo1 = load_balance_get_bucket_i (lb1, 0);
 837         }
 838
 839       next[0] = dpo0->dpoi_next_node;
 840       next[1] = dpo1->dpoi_next_node;
 841
 842       /* Only process the HBH Option Header if explicitly configured to do so */
 843       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 844         {
 845           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 846             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 847         }
 848       /* Only process the HBH Option Header if explicitly configured to do so */
 849       if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 850         {
 851           next[1] = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 852             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[1];
 853         }
 854
 855       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 856       vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 857
 858       vlib_increment_combined_counter
 859         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 860       vlib_increment_combined_counter
 861         (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b[1]));
 862
 863       b += 2;
 864       next += 2;
 865       n_left -= 2;
 866     }
 867
 868   while (n_left > 0)
 869     {
 870       const load_balance_t *lb0;
 871       const ip6_header_t *ip0;
 872       const dpo_id_t *dpo0;
 873       u32 lbi0, hc0;
 874
 875       ip0 = vlib_buffer_get_current (b[0]);
 876       lbi0 = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
 877
 878       lb0 = load_balance_get (lbi0);
 879
 880       hc0 = 0;
 881       if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 882         {
 883           if (PREDICT_TRUE (vnet_buffer (b[0])->ip.flow_hash))
 884             {
 885               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 886                 vnet_buffer (b[0])->ip.flow_hash >> 1;
 887             }
 888           else
 889             {
 890               hc0 = vnet_buffer (b[0])->ip.flow_hash =
 891                 ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 892             }
 893           dpo0 = load_balance_get_fwd_bucket
 894             (lb0, (hc0 & (lb0->lb_n_buckets_minus_1)));
 895         }
 896       else
 897         {
 898           dpo0 = load_balance_get_bucket_i (lb0, 0);
 899         }
 900
 901       next[0] = dpo0->dpoi_next_node;
 902       vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 903
 904       /* Only process the HBH Option Header if explicitly configured to do so */
 905       if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 906         {
 907           next[0] = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 908             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next[0];
 909         }
 910
 911       vlib_increment_combined_counter
 912         (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b[0]));
 913
 914       b += 1;
 915       next += 1;
 916       n_left -= 1;
 917     }
 918
 919   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
 920
 921   if (node->flags & VLIB_NODE_FLAG_TRACE)
 922     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 923
 924   return frame->n_vectors;
 925 }
 926
 927 /* *INDENT-OFF* */
 928 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 929 {
 930   .name = "ip6-load-balance",
 931   .vector_size = sizeof (u32),
 932   .sibling_of = "ip6-lookup",
 933   .format_trace = format_ip6_lookup_trace,
 934 };
 935 /* *INDENT-ON* */
 936
 937 typedef struct
 938 {
 939   /* Adjacency taken. */
 940   u32 adj_index;
 941   u32 flow_hash;
 942   u32 fib_index;
 943
 944   /* Packet data, possibly *after* rewrite. */
 945   u8 packet_data[128 - 1 * sizeof (u32)];
 946 }
 947 ip6_forward_next_trace_t;
 948
 949 #ifndef CLIB_MARCH_VARIANT
 950 u8 *
 951 format_ip6_forward_next_trace (u8 * s, va_list * args)
 952 {
 953   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 954   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 955   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 956   u32 indent = format_get_indent (s);
 957
 958   s = format (s, "%U%U",
 959               format_white_space, indent,
 960               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 961   return s;
 962 }
 963 #endif
 964
 965 static u8 *
 966 format_ip6_lookup_trace (u8 * s, va_list * args)
 967 {
 968   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 969   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 970   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 971   u32 indent = format_get_indent (s);
 972
 973   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 974               t->fib_index, t->adj_index, t->flow_hash);
 975   s = format (s, "\n%U%U",
 976               format_white_space, indent,
 977               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 978   return s;
 979 }
 980
 981
 982 static u8 *
 983 format_ip6_rewrite_trace (u8 * s, va_list * args)
 984 {
 985   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 986   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 987   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 988   u32 indent = format_get_indent (s);
 989
 990   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 991               t->fib_index, t->adj_index, format_ip_adjacency,
 992               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 993   s = format (s, "\n%U%U",
 994               format_white_space, indent,
 995               format_ip_adjacency_packet_data,
 996               t->packet_data, sizeof (t->packet_data));
 997   return s;
 998 }
 999
1000 /* Common trace function for all ip6-forward next nodes. */
1001 #ifndef CLIB_MARCH_VARIANT
1002 void
1003 ip6_forward_next_trace (vlib_main_t * vm,
1004                         vlib_node_runtime_t * node,
1005                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1006 {
1007   u32 *from, n_left;
1008   ip6_main_t *im = &ip6_main;
1009
1010   n_left = frame->n_vectors;
1011   from = vlib_frame_vector_args (frame);
1012
1013   while (n_left >= 4)
1014     {
1015       u32 bi0, bi1;
1016       vlib_buffer_t *b0, *b1;
1017       ip6_forward_next_trace_t *t0, *t1;
1018
1019       /* Prefetch next iteration. */
1020       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1021       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1022
1023       bi0 = from[0];
1024       bi1 = from[1];
1025
1026       b0 = vlib_get_buffer (vm, bi0);
1027       b1 = vlib_get_buffer (vm, bi1);
1028
1029       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1030         {
1031           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1032           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1033           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1034           t0->fib_index =
1035             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1036              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1037             vec_elt (im->fib_index_by_sw_if_index,
1038                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1039
1040           clib_memcpy_fast (t0->packet_data,
1041                             vlib_buffer_get_current (b0),
1042                             sizeof (t0->packet_data));
1043         }
1044       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1045         {
1046           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1047           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1048           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1049           t1->fib_index =
1050             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1051              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1052             vec_elt (im->fib_index_by_sw_if_index,
1053                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1054
1055           clib_memcpy_fast (t1->packet_data,
1056                             vlib_buffer_get_current (b1),
1057                             sizeof (t1->packet_data));
1058         }
1059       from += 2;
1060       n_left -= 2;
1061     }
1062
1063   while (n_left >= 1)
1064     {
1065       u32 bi0;
1066       vlib_buffer_t *b0;
1067       ip6_forward_next_trace_t *t0;
1068
1069       bi0 = from[0];
1070
1071       b0 = vlib_get_buffer (vm, bi0);
1072
1073       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1074         {
1075           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1076           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1077           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1078           t0->fib_index =
1079             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1080              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1081             vec_elt (im->fib_index_by_sw_if_index,
1082                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1083
1084           clib_memcpy_fast (t0->packet_data,
1085                             vlib_buffer_get_current (b0),
1086                             sizeof (t0->packet_data));
1087         }
1088       from += 1;
1089       n_left -= 1;
1090     }
1091 }
1092
1093 /* Compute TCP/UDP/ICMP6 checksum in software. */
1094 u16
1095 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
1096                                    ip6_header_t * ip0, int *bogus_lengthp)
1097 {
1098   ip_csum_t sum0 = 0;
1099   u16 payload_length, payload_length_host_byte_order;
1100   u32 i;
1101   u32 headers_size = sizeof (ip0[0]);
1102   u8 *data_this_buffer;
1103   u8 next_hdr = ip0->protocol;
1104
1105   ASSERT (bogus_lengthp);
1106   *bogus_lengthp = 0;
1107
1108   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1109   data_this_buffer = (u8 *) (ip0 + 1);
1110   payload_length = ip0->payload_length;
1111
1112   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
1113    * or UDP-Ping packets */
1114   if (PREDICT_FALSE (next_hdr == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1115     {
1116       u32 skip_bytes;
1117       ip6_hop_by_hop_ext_t *ext_hdr =
1118         (ip6_hop_by_hop_ext_t *) data_this_buffer;
1119
1120       /* validate really icmp6 next */
1121       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1122               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
1123
1124       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
1125       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
1126
1127       payload_length_host_byte_order -= skip_bytes;
1128       headers_size += skip_bytes;
1129
1130       /* pseudo-header adjustments:
1131        *   exclude ext header bytes from payload length
1132        *   use payload IP proto rather than ext header IP proto
1133        */
1134       payload_length = clib_host_to_net_u16 (payload_length_host_byte_order);
1135       next_hdr = ext_hdr->next_hdr;
1136     }
1137
1138   /* Initialize checksum with ip pseudo-header. */
1139   sum0 = payload_length + clib_host_to_net_u16 (next_hdr);
1140
1141   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1142     {
1143       sum0 = ip_csum_with_carry
1144         (sum0, clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1145       sum0 = ip_csum_with_carry
1146         (sum0, clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1147     }
1148
1149   if (p0)
1150     return ip_calculate_l4_checksum (vm, p0, sum0,
1151                                      payload_length_host_byte_order,
1152                                      (u8 *) ip0, headers_size, NULL);
1153   else
1154     return ip_calculate_l4_checksum (vm, 0, sum0,
1155                                      payload_length_host_byte_order, NULL, 0,
1156                                      data_this_buffer);
1157 }
1158
1159 u32
1160 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1161 {
1162   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1163   udp_header_t *udp0;
1164   u16 sum16;
1165   int bogus_length;
1166
1167   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1168   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1169           || ip0->protocol == IP_PROTOCOL_ICMP6
1170           || ip0->protocol == IP_PROTOCOL_UDP
1171           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1172
1173   udp0 = (void *) (ip0 + 1);
1174   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1175     {
1176       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1177                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
1178       return p0->flags;
1179     }
1180
1181   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1182
1183   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
1184                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
1185
1186   return p0->flags;
1187 }
1188 #endif
1189
1190 /**
1191  * @brief returns number of links on which src is reachable.
1192  */
1193 always_inline int
1194 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
1195 {
1196   const load_balance_t *lb0;
1197   index_t lbi;
1198   u32 fib_index;
1199
1200   fib_index = vec_elt (im->fib_index_by_sw_if_index,
1201                        vnet_buffer (b)->sw_if_index[VLIB_RX]);
1202   fib_index =
1203     (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1204     fib_index : vnet_buffer (b)->sw_if_index[VLIB_TX];
1205
1206   lbi = ip6_fib_table_fwding_lookup (fib_index, &i->src_address);
1207   lb0 = load_balance_get (lbi);
1208
1209   return (fib_urpf_check_size (lb0->lb_urpf));
1210 }
1211
1212 always_inline u8
1213 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
1214                            u32 * udp_offset0)
1215 {
1216   u32 proto0;
1217   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
1218   if (proto0 != IP_PROTOCOL_UDP)
1219     {
1220       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
1221       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
1222     }
1223   return proto0;
1224 }
1225
1226 /* *INDENT-OFF* */
1227 VNET_FEATURE_ARC_INIT (ip6_local) =
1228 {
1229   .arc_name  = "ip6-local",
1230   .start_nodes = VNET_FEATURES ("ip6-local"),
1231 };
1232 /* *INDENT-ON* */
1233
1234 static_always_inline u8
1235 ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
1236 {
1237
1238   u16 payload_length_host_byte_order;
1239   u32 n_this_buffer, n_bytes_left;
1240   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
1241   u32 headers_size = sizeof (ip0[0]);
1242   u8 *data_this_buffer;
1243
1244
1245   data_this_buffer = (u8 *) (ip0 + 1);
1246
1247   ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t *) data_this_buffer;
1248
1249   /* validate really icmp6 next */
1250
1251   if (!(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
1252       || (ext_hdr->next_hdr == IP_PROTOCOL_UDP))
1253     return 0;
1254
1255
1256   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1257   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1258
1259
1260   u32 n_ip_bytes_this_buffer =
1261     p0->current_length - (((u8 *) ip0 - p0->data) - p0->current_data);
1262   if (n_this_buffer + headers_size > n_ip_bytes_this_buffer)
1263     {
1264       n_this_buffer = p0->current_length > headers_size ?
1265         n_ip_bytes_this_buffer - headers_size : 0;
1266     }
1267
1268   n_bytes_left -= n_this_buffer;
1269   n_bytes_left -= p0->total_length_not_including_first_buffer;
1270
1271   if (n_bytes_left == 0)
1272     return 0;
1273   else
1274     return 1;
1275 }
1276
1277
1278 always_inline uword
1279 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1280                   vlib_frame_t * frame, int head_of_feature_arc)
1281 {
1282   ip6_main_t *im = &ip6_main;
1283   ip_lookup_main_t *lm = &im->lookup_main;
1284   u32 *from, n_left_from;
1285   vlib_node_runtime_t *error_node =
1286     vlib_node_get_runtime (vm, ip6_input_node.index);
1287   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1288   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1289   u16 nexts[VLIB_FRAME_SIZE], *next;
1290
1291   from = vlib_frame_vector_args (frame);
1292   n_left_from = frame->n_vectors;
1293
1294   if (node->flags & VLIB_NODE_FLAG_TRACE)
1295     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1296
1297   vlib_get_buffers (vm, from, bufs, n_left_from);
1298   b = bufs;
1299   next = nexts;
1300
1301   while (n_left_from > 2)
1302     {
1303       /* Prefetch next iteration. */
1304       if (n_left_from >= 6)
1305         {
1306           vlib_prefetch_buffer_header (b[4], STORE);
1307           vlib_prefetch_buffer_header (b[5], STORE);
1308           vlib_prefetch_buffer_data (b[2], LOAD);
1309           vlib_prefetch_buffer_data (b[3], LOAD);
1310         }
1311
1312       u8 error[2];
1313       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1314       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1315
1316       ip6_header_t *ip[2];
1317       ip[0] = vlib_buffer_get_current (b[0]);
1318       ip[1] = vlib_buffer_get_current (b[1]);
1319
1320       if (head_of_feature_arc)
1321         {
1322           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1323           vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data;
1324
1325           u8 type[2];
1326           type[0] = lm->builtin_protocol_by_ip_protocol[ip[0]->protocol];
1327           type[1] = lm->builtin_protocol_by_ip_protocol[ip[1]->protocol];
1328
1329           u32 flags[2];
1330           flags[0] = b[0]->flags;
1331           flags[1] = b[1]->flags;
1332
1333           u32 good_l4_csum[2];
1334           good_l4_csum[0] =
1335             flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1336                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1337                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1338           good_l4_csum[1] =
1339             flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1340                         VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1341                         VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1342
1343           u32 udp_offset[2] = { };
1344           u8 is_tcp_udp[2];
1345           is_tcp_udp[0] =
1346             ip6_next_proto_is_tcp_udp (b[0], ip[0], &udp_offset[0]);
1347           is_tcp_udp[1] =
1348             ip6_next_proto_is_tcp_udp (b[1], ip[1], &udp_offset[1]);
1349           i16 len_diff[2] = { 0 };
1350           if (PREDICT_TRUE (is_tcp_udp[0]))
1351             {
1352               udp_header_t *udp =
1353                 (udp_header_t *) ((u8 *) ip[0] + udp_offset[0]);
1354               good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UDP
1355                 && udp->checksum == 0;
1356               /* optimistically verify UDP length. */
1357               u16 ip_len, udp_len;
1358               ip_len = clib_net_to_host_u16 (ip[0]->payload_length);
1359               udp_len = clib_net_to_host_u16 (udp->length);
1360               len_diff[0] = ip_len - udp_len;
1361             }
1362           if (PREDICT_TRUE (is_tcp_udp[1]))
1363             {
1364               udp_header_t *udp =
1365                 (udp_header_t *) ((u8 *) ip[1] + udp_offset[1]);
1366               good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UDP
1367                 && udp->checksum == 0;
1368               /* optimistically verify UDP length. */
1369               u16 ip_len, udp_len;
1370               ip_len = clib_net_to_host_u16 (ip[1]->payload_length);
1371               udp_len = clib_net_to_host_u16 (udp->length);
1372               len_diff[1] = ip_len - udp_len;
1373             }
1374
1375           good_l4_csum[0] |= type[0] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1376           good_l4_csum[1] |= type[1] == IP_BUILTIN_PROTOCOL_UNKNOWN;
1377
1378           len_diff[0] = type[0] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[0] : 0;
1379           len_diff[1] = type[1] == IP_BUILTIN_PROTOCOL_UDP ? len_diff[1] : 0;
1380
1381           u8 need_csum[2];
1382           need_csum[0] = type[0] != IP_BUILTIN_PROTOCOL_UNKNOWN
1383             && !good_l4_csum[0]
1384             && !(flags[0] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1385           need_csum[1] = type[1] != IP_BUILTIN_PROTOCOL_UNKNOWN
1386             && !good_l4_csum[1]
1387             && !(flags[1] & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1388           if (PREDICT_FALSE (need_csum[0]))
1389             {
1390               flags[0] = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1391               good_l4_csum[0] = flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1392               error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
1393             }
1394           else
1395             {
1396               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1397                 error[0] = IP6_ERROR_BAD_LENGTH;
1398             }
1399           if (PREDICT_FALSE (need_csum[1]))
1400             {
1401               flags[1] = ip6_tcp_udp_icmp_validate_checksum (vm, b[1]);
1402               good_l4_csum[1] = flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1403               error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
1404             }
1405           else
1406             {
1407               if (ip6_tcp_udp_icmp_bad_length (vm, b[1]))
1408                 error[1] = IP6_ERROR_BAD_LENGTH;
1409             }
1410
1411
1412           error[0] = len_diff[0] < 0 ? IP6_ERROR_UDP_LENGTH : error[0];
1413
1414           error[1] = len_diff[1] < 0 ? IP6_ERROR_UDP_LENGTH : error[1];
1415
1416           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1417                          IP6_ERROR_UDP_CHECKSUM,
1418                          "Wrong IP6 errors constants");
1419           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1420                          IP6_ERROR_ICMP_CHECKSUM,
1421                          "Wrong IP6 errors constants");
1422
1423           error[0] =
1424             !good_l4_csum[0] ? IP6_ERROR_UDP_CHECKSUM + type[0] : error[0];
1425           error[1] =
1426             !good_l4_csum[1] ? IP6_ERROR_UDP_CHECKSUM + type[1] : error[1];
1427
1428           /* Drop packets from unroutable hosts. */
1429           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1430           u8 unroutable[2];
1431           unroutable[0] = error[0] == IP6_ERROR_UNKNOWN_PROTOCOL
1432             && type[0] != IP_BUILTIN_PROTOCOL_ICMP
1433             && !ip6_address_is_link_local_unicast (&ip[0]->src_address);
1434           unroutable[1] = error[1] == IP6_ERROR_UNKNOWN_PROTOCOL
1435             && type[1] != IP_BUILTIN_PROTOCOL_ICMP
1436             && !ip6_address_is_link_local_unicast (&ip[1]->src_address);
1437           if (PREDICT_FALSE (unroutable[0]))
1438             {
1439               error[0] =
1440                 !ip6_urpf_loose_check (im, b[0],
1441                                        ip[0]) ? IP6_ERROR_SRC_LOOKUP_MISS
1442                 : error[0];
1443             }
1444           if (PREDICT_FALSE (unroutable[1]))
1445             {
1446               error[1] =
1447                 !ip6_urpf_loose_check (im, b[1],
1448                                        ip[1]) ? IP6_ERROR_SRC_LOOKUP_MISS
1449                 : error[1];
1450             }
1451
1452           vnet_buffer (b[0])->ip.fib_index =
1453             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1454             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1455             vnet_buffer (b[0])->ip.fib_index;
1456           vnet_buffer (b[1])->ip.fib_index =
1457             vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
1458             vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
1459             vnet_buffer (b[1])->ip.fib_index;
1460         }                       /* head_of_feature_arc */
1461
1462       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
1463       next[0] =
1464         error[0] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1465       next[1] = lm->local_next_by_ip_protocol[ip[1]->protocol];
1466       next[1] =
1467         error[1] != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[1];
1468
1469       b[0]->error = error_node->errors[0];
1470       b[1]->error = error_node->errors[1];
1471
1472       if (head_of_feature_arc)
1473         {
1474           u8 ip6_unknown[2];
1475           ip6_unknown[0] = error[0] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1476           ip6_unknown[1] = error[1] == (u8) IP6_ERROR_UNKNOWN_PROTOCOL;
1477           if (PREDICT_TRUE (ip6_unknown[0]))
1478             {
1479               u32 next32 = next[0];
1480               vnet_feature_arc_start (arc_index,
1481                                       vnet_buffer (b[0])->sw_if_index
1482                                       [VLIB_RX], &next32, b[0]);
1483               next[0] = next32;
1484             }
1485           if (PREDICT_TRUE (ip6_unknown[1]))
1486             {
1487               u32 next32 = next[1];
1488               vnet_feature_arc_start (arc_index,
1489                                       vnet_buffer (b[1])->sw_if_index
1490                                       [VLIB_RX], &next32, b[1]);
1491               next[1] = next32;
1492             }
1493         }
1494
1495       /* next */
1496       b += 2;
1497       next += 2;
1498       n_left_from -= 2;
1499     }
1500
1501   while (n_left_from)
1502     {
1503       u8 error;
1504       error = IP6_ERROR_UNKNOWN_PROTOCOL;
1505
1506       ip6_header_t *ip;
1507       ip = vlib_buffer_get_current (b[0]);
1508
1509       if (head_of_feature_arc)
1510         {
1511           vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data;
1512           u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
1513
1514           u32 flags = b[0]->flags;
1515           u32 good_l4_csum =
1516             flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
1517                      VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
1518                      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
1519
1520           u32 udp_offset;
1521           i16 len_diff = 0;
1522           u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
1523           if (PREDICT_TRUE (is_tcp_udp))
1524             {
1525               udp_header_t *udp = (udp_header_t *) ((u8 *) ip + udp_offset);
1526               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1527               good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UDP
1528                 && udp->checksum == 0;
1529               /* optimistically verify UDP length. */
1530               u16 ip_len, udp_len;
1531               ip_len = clib_net_to_host_u16 (ip->payload_length);
1532               udp_len = clib_net_to_host_u16 (udp->length);
1533               len_diff = ip_len - udp_len;
1534             }
1535
1536           good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
1537           len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
1538
1539           u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
1540             && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
1541           if (PREDICT_FALSE (need_csum))
1542             {
1543               flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
1544               good_l4_csum = flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
1545               error = IP6_ERROR_UNKNOWN_PROTOCOL;
1546             }
1547           else
1548             {
1549               if (ip6_tcp_udp_icmp_bad_length (vm, b[0]))
1550                 error = IP6_ERROR_BAD_LENGTH;
1551             }
1552
1553
1554
1555           error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
1556
1557           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1558                          IP6_ERROR_UDP_CHECKSUM,
1559                          "Wrong IP6 errors constants");
1560           STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1561                          IP6_ERROR_ICMP_CHECKSUM,
1562                          "Wrong IP6 errors constants");
1563
1564           error = !good_l4_csum ? IP6_ERROR_UDP_CHECKSUM + type : error;
1565
1566           /* Drop packets from unroutable hosts. */
1567           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1568           u8 unroutable = error == IP6_ERROR_UNKNOWN_PROTOCOL
1569             && type != IP_BUILTIN_PROTOCOL_ICMP
1570             && !ip6_address_is_link_local_unicast (&ip->src_address);
1571           if (PREDICT_FALSE (unroutable))
1572             {
1573               error =
1574                 !ip6_urpf_loose_check (im, b[0],
1575                                        ip) ? IP6_ERROR_SRC_LOOKUP_MISS :
1576                 error;
1577             }
1578
1579           vnet_buffer (b[0])->ip.fib_index =
1580             vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
1581             vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
1582             vnet_buffer (b[0])->ip.fib_index;
1583         }                       /* head_of_feature_arc */
1584
1585       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
1586       next[0] =
1587         error != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next[0];
1588
1589       b[0]->error = error_node->errors[0];
1590
1591       if (head_of_feature_arc)
1592         {
1593           if (PREDICT_TRUE (error == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1594             {
1595               u32 next32 = next[0];
1596               vnet_feature_arc_start (arc_index,
1597                                       vnet_buffer (b[0])->sw_if_index
1598                                       [VLIB_RX], &next32, b[0]);
1599               next[0] = next32;
1600             }
1601         }
1602
1603       /* next */
1604       b += 1;
1605       next += 1;
1606       n_left_from -= 1;
1607     }
1608
1609   vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
1610   return frame->n_vectors;
1611 }
1612
1613 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
1614                                vlib_frame_t * frame)
1615 {
1616   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1617 }
1618
1619 /* *INDENT-OFF* */
1620 VLIB_REGISTER_NODE (ip6_local_node) =
1621 {
1622   .name = "ip6-local",
1623   .vector_size = sizeof (u32),
1624   .format_trace = format_ip6_forward_next_trace,
1625   .n_next_nodes = IP_LOCAL_N_NEXT,
1626   .next_nodes =
1627   {
1628     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1629     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1630     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1631     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1632     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-full-reassembly",
1633   },
1634 };
1635 /* *INDENT-ON* */
1636
1637 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
1638                                           vlib_node_runtime_t * node,
1639                                           vlib_frame_t * frame)
1640 {
1641   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1642 }
1643
1644 /* *INDENT-OFF* */
1645 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
1646   .name = "ip6-local-end-of-arc",
1647   .vector_size = sizeof (u32),
1648
1649   .format_trace = format_ip6_forward_next_trace,
1650   .sibling_of = "ip6-local",
1651 };
1652
1653 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1654   .arc_name = "ip6-local",
1655   .node_name = "ip6-local-end-of-arc",
1656   .runs_before = 0, /* not before any other features */
1657 };
1658 /* *INDENT-ON* */
1659
1660 #ifdef CLIB_MARCH_VARIANT
1661 extern vlib_node_registration_t ip6_local_node;
1662 #else
1663 void
1664 ip6_register_protocol (u32 protocol, u32 node_index)
1665 {
1666   vlib_main_t *vm = vlib_get_main ();
1667   ip6_main_t *im = &ip6_main;
1668   ip_lookup_main_t *lm = &im->lookup_main;
1669
1670   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1671   lm->local_next_by_ip_protocol[protocol] =
1672     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1673 }
1674
1675 void
1676 ip6_unregister_protocol (u32 protocol)
1677 {
1678   ip6_main_t *im = &ip6_main;
1679   ip_lookup_main_t *lm = &im->lookup_main;
1680
1681   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1682   lm->local_next_by_ip_protocol[protocol] = IP_LOCAL_NEXT_PUNT;
1683 }
1684 #endif
1685
1686 typedef enum
1687 {
1688   IP6_REWRITE_NEXT_DROP,
1689   IP6_REWRITE_NEXT_ICMP_ERROR,
1690   IP6_REWRITE_NEXT_FRAGMENT,
1691   IP6_REWRITE_N_NEXT            /* Last */
1692 } ip6_rewrite_next_t;
1693
1694 /**
1695  * This bits of an IPv6 address to mask to construct a multicast
1696  * MAC address
1697  */
1698 #define IP6_MCAST_ADDR_MASK 0xffffffff
1699
1700 always_inline void
1701 ip6_mtu_check (vlib_buffer_t * b, u16 packet_bytes,
1702                u16 adj_packet_bytes, bool is_locally_generated,
1703                u32 * next, u8 is_midchain, u32 * error)
1704 {
1705   if (adj_packet_bytes >= 1280 && packet_bytes > adj_packet_bytes)
1706     {
1707       if (is_locally_generated)
1708         {
1709           /* IP fragmentation */
1710           ip_frag_set_vnet_buffer (b, adj_packet_bytes,
1711                                    (is_midchain ?
1712                                     IP_FRAG_NEXT_IP_REWRITE_MIDCHAIN :
1713                                     IP_FRAG_NEXT_IP_REWRITE), 0);
1714           *next = IP6_REWRITE_NEXT_FRAGMENT;
1715           *error = IP6_ERROR_MTU_EXCEEDED;
1716         }
1717       else
1718         {
1719           *error = IP6_ERROR_MTU_EXCEEDED;
1720           icmp6_error_set_vnet_buffer (b, ICMP6_packet_too_big, 0,
1721                                        adj_packet_bytes);
1722           *next = IP6_REWRITE_NEXT_ICMP_ERROR;
1723         }
1724     }
1725 }
1726
1727 always_inline uword
1728 ip6_rewrite_inline_with_gso (vlib_main_t * vm,
1729                              vlib_node_runtime_t * node,
1730                              vlib_frame_t * frame,
1731                              int do_counters, int is_midchain, int is_mcast)
1732 {
1733   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1734   u32 *from = vlib_frame_vector_args (frame);
1735   u32 n_left_from, n_left_to_next, *to_next, next_index;
1736   vlib_node_runtime_t *error_node =
1737     vlib_node_get_runtime (vm, ip6_input_node.index);
1738
1739   n_left_from = frame->n_vectors;
1740   next_index = node->cached_next_index;
1741   u32 thread_index = vm->thread_index;
1742
1743   while (n_left_from > 0)
1744     {
1745       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1746
1747       while (n_left_from >= 4 && n_left_to_next >= 2)
1748         {
1749           const ip_adjacency_t *adj0, *adj1;
1750           vlib_buffer_t *p0, *p1;
1751           ip6_header_t *ip0, *ip1;
1752           u32 pi0, rw_len0, next0, error0, adj_index0;
1753           u32 pi1, rw_len1, next1, error1, adj_index1;
1754           u32 tx_sw_if_index0, tx_sw_if_index1;
1755           bool is_locally_originated0, is_locally_originated1;
1756
1757           /* Prefetch next iteration. */
1758           {
1759             vlib_buffer_t *p2, *p3;
1760
1761             p2 = vlib_get_buffer (vm, from[2]);
1762             p3 = vlib_get_buffer (vm, from[3]);
1763
1764             vlib_prefetch_buffer_header (p2, LOAD);
1765             vlib_prefetch_buffer_header (p3, LOAD);
1766
1767             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1768             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1769
1770             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1771             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1772           }
1773
1774           pi0 = to_next[0] = from[0];
1775           pi1 = to_next[1] = from[1];
1776
1777           from += 2;
1778           n_left_from -= 2;
1779           to_next += 2;
1780           n_left_to_next -= 2;
1781
1782           p0 = vlib_get_buffer (vm, pi0);
1783           p1 = vlib_get_buffer (vm, pi1);
1784
1785           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1786           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1787
1788           ip0 = vlib_buffer_get_current (p0);
1789           ip1 = vlib_buffer_get_current (p1);
1790
1791           error0 = error1 = IP6_ERROR_NONE;
1792           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1793
1794           is_locally_originated0 =
1795             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1796           if (PREDICT_TRUE (!is_locally_originated0))
1797             {
1798               i32 hop_limit0 = ip0->hop_limit;
1799
1800               /* Input node should have reject packets with hop limit 0. */
1801               ASSERT (ip0->hop_limit > 0);
1802
1803               hop_limit0 -= 1;
1804
1805               ip0->hop_limit = hop_limit0;
1806
1807               /*
1808                * If the hop count drops below 1 when forwarding, generate
1809                * an ICMP response.
1810                */
1811               if (PREDICT_FALSE (hop_limit0 <= 0))
1812                 {
1813                   error0 = IP6_ERROR_TIME_EXPIRED;
1814                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1815                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1816                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1817                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1818                                                0);
1819                 }
1820             }
1821
1822           is_locally_originated1 =
1823             p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
1824           if (PREDICT_TRUE (!is_locally_originated1))
1825             {
1826               i32 hop_limit1 = ip1->hop_limit;
1827
1828               /* Input node should have reject packets with hop limit 0. */
1829               ASSERT (ip1->hop_limit > 0);
1830
1831               hop_limit1 -= 1;
1832
1833               ip1->hop_limit = hop_limit1;
1834
1835               /*
1836                * If the hop count drops below 1 when forwarding, generate
1837                * an ICMP response.
1838                */
1839               if (PREDICT_FALSE (hop_limit1 <= 0))
1840                 {
1841                   error1 = IP6_ERROR_TIME_EXPIRED;
1842                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1843                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1844                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1845                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1846                                                0);
1847                 }
1848             }
1849
1850           adj0 = adj_get (adj_index0);
1851           adj1 = adj_get (adj_index1);
1852
1853           rw_len0 = adj0[0].rewrite_header.data_bytes;
1854           rw_len1 = adj1[0].rewrite_header.data_bytes;
1855           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1856           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1857
1858           if (do_counters)
1859             {
1860               vlib_increment_combined_counter
1861                 (&adjacency_counters,
1862                  thread_index, adj_index0, 1,
1863                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1864               vlib_increment_combined_counter
1865                 (&adjacency_counters,
1866                  thread_index, adj_index1, 1,
1867                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1868             }
1869
1870           /* Check MTU of outgoing interface. */
1871           u16 ip0_len =
1872             clib_net_to_host_u16 (ip0->payload_length) +
1873             sizeof (ip6_header_t);
1874           u16 ip1_len =
1875             clib_net_to_host_u16 (ip1->payload_length) +
1876             sizeof (ip6_header_t);
1877           if (p0->flags & VNET_BUFFER_F_GSO)
1878             ip0_len = gso_mtu_sz (p0);
1879           if (p1->flags & VNET_BUFFER_F_GSO)
1880             ip1_len = gso_mtu_sz (p1);
1881
1882
1883
1884           ip6_mtu_check (p0, ip0_len,
1885                          adj0[0].rewrite_header.max_l3_packet_bytes,
1886                          is_locally_originated0, &next0, is_midchain,
1887                          &error0);
1888           ip6_mtu_check (p1, ip1_len,
1889                          adj1[0].rewrite_header.max_l3_packet_bytes,
1890                          is_locally_originated1, &next1, is_midchain,
1891                          &error1);
1892
1893           /* Don't adjust the buffer for hop count issue; icmp-error node
1894            * wants to see the IP header */
1895           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1896             {
1897               p0->current_data -= rw_len0;
1898               p0->current_length += rw_len0;
1899
1900               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1901               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1902               next0 = adj0[0].rewrite_header.next_index;
1903
1904               if (PREDICT_FALSE
1905                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1906                 vnet_feature_arc_start_w_cfg_index
1907                   (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0,
1908                    adj0->ia_cfg_index);
1909             }
1910           else
1911             {
1912               p0->error = error_node->errors[error0];
1913             }
1914           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1915             {
1916               p1->current_data -= rw_len1;
1917               p1->current_length += rw_len1;
1918
1919               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1920               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1921               next1 = adj1[0].rewrite_header.next_index;
1922
1923               if (PREDICT_FALSE
1924                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1925                 vnet_feature_arc_start_w_cfg_index
1926                   (lm->output_feature_arc_index, tx_sw_if_index1, &next1, p1,
1927                    adj1->ia_cfg_index);
1928             }
1929           else
1930             {
1931               p1->error = error_node->errors[error1];
1932             }
1933
1934           if (is_midchain)
1935             {
1936               /* before we paint on the next header, update the L4
1937                * checksums if required, since there's no offload on a tunnel */
1938               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
1939                                           1 /* is_ip6 */ ,
1940                                           0 /* with gso */ );
1941               vnet_calc_checksums_inline (vm, p1, 0 /* is_ip4 */ ,
1942                                           1 /* is_ip6 */ ,
1943                                           0 /* with gso */ );
1944
1945               /* Guess we are only writing on ipv6 header. */
1946               vnet_rewrite_two_headers (adj0[0], adj1[0],
1947                                         ip0, ip1, sizeof (ip6_header_t));
1948             }
1949           else
1950             /* Guess we are only writing on simple Ethernet header. */
1951             vnet_rewrite_two_headers (adj0[0], adj1[0],
1952                                       ip0, ip1, sizeof (ethernet_header_t));
1953
1954           if (is_midchain)
1955             {
1956               if (adj0->sub_type.midchain.fixup_func)
1957                 adj0->sub_type.midchain.fixup_func
1958                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1959               if (adj1->sub_type.midchain.fixup_func)
1960                 adj1->sub_type.midchain.fixup_func
1961                   (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1962             }
1963           if (is_mcast)
1964             {
1965               /*
1966                * copy bytes from the IP address into the MAC rewrite
1967                */
1968               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1969                                           adj0->
1970                                           rewrite_header.dst_mcast_offset,
1971                                           &ip0->dst_address.as_u32[3],
1972                                           (u8 *) ip0);
1973               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
1974                                           adj1->
1975                                           rewrite_header.dst_mcast_offset,
1976                                           &ip1->dst_address.as_u32[3],
1977                                           (u8 *) ip1);
1978             }
1979
1980           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1981                                            to_next, n_left_to_next,
1982                                            pi0, pi1, next0, next1);
1983         }
1984
1985       while (n_left_from > 0 && n_left_to_next > 0)
1986         {
1987           ip_adjacency_t *adj0;
1988           vlib_buffer_t *p0;
1989           ip6_header_t *ip0;
1990           u32 pi0, rw_len0;
1991           u32 adj_index0, next0, error0;
1992           u32 tx_sw_if_index0;
1993           bool is_locally_originated0;
1994
1995           pi0 = to_next[0] = from[0];
1996
1997           p0 = vlib_get_buffer (vm, pi0);
1998
1999           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2000
2001           adj0 = adj_get (adj_index0);
2002
2003           ip0 = vlib_buffer_get_current (p0);
2004
2005           error0 = IP6_ERROR_NONE;
2006           next0 = IP6_REWRITE_NEXT_DROP;
2007
2008           /* Check hop limit */
2009           is_locally_originated0 =
2010             p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED;
2011           if (PREDICT_TRUE (!is_locally_originated0))
2012             {
2013               i32 hop_limit0 = ip0->hop_limit;
2014
2015               ASSERT (ip0->hop_limit > 0);
2016
2017               hop_limit0 -= 1;
2018
2019               ip0->hop_limit = hop_limit0;
2020
2021               if (PREDICT_FALSE (hop_limit0 <= 0))
2022                 {
2023                   /*
2024                    * If the hop count drops below 1 when forwarding, generate
2025                    * an ICMP response.
2026                    */
2027                   error0 = IP6_ERROR_TIME_EXPIRED;
2028                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2029                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2030                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
2031                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
2032                                                0);
2033                 }
2034             }
2035
2036           if (is_midchain)
2037             {
2038               vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
2039                                           1 /* is_ip6 */ ,
2040                                           0 /* with gso */ );
2041
2042               /* Guess we are only writing on ip6 header. */
2043               vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip6_header_t));
2044             }
2045           else
2046             /* Guess we are only writing on simple Ethernet header. */
2047             vnet_rewrite_one_header (adj0[0], ip0,
2048                                      sizeof (ethernet_header_t));
2049
2050           /* Update packet buffer attributes/set output interface. */
2051           rw_len0 = adj0[0].rewrite_header.data_bytes;
2052           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2053
2054           if (do_counters)
2055             {
2056               vlib_increment_combined_counter
2057                 (&adjacency_counters,
2058                  thread_index, adj_index0, 1,
2059                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2060             }
2061
2062           /* Check MTU of outgoing interface. */
2063           u16 ip0_len =
2064             clib_net_to_host_u16 (ip0->payload_length) +
2065             sizeof (ip6_header_t);
2066           if (p0->flags & VNET_BUFFER_F_GSO)
2067             ip0_len = gso_mtu_sz (p0);
2068
2069           ip6_mtu_check (p0, ip0_len,
2070                          adj0[0].rewrite_header.max_l3_packet_bytes,
2071                          is_locally_originated0, &next0, is_midchain,
2072                          &error0);
2073
2074           /* Don't adjust the buffer for hop count issue; icmp-error node
2075            * wants to see the IP header */
2076           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
2077             {
2078               p0->current_data -= rw_len0;
2079               p0->current_length += rw_len0;
2080
2081               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2082
2083               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2084               next0 = adj0[0].rewrite_header.next_index;
2085
2086               if (PREDICT_FALSE
2087                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2088                 vnet_feature_arc_start_w_cfg_index
2089                   (lm->output_feature_arc_index, tx_sw_if_index0, &next0, p0,
2090                    adj0->ia_cfg_index);
2091             }
2092           else
2093             {
2094               p0->error = error_node->errors[error0];
2095             }
2096
2097           if (is_midchain)
2098             {
2099               if (adj0->sub_type.midchain.fixup_func)
2100                 adj0->sub_type.midchain.fixup_func
2101                   (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2102             }
2103           if (is_mcast)
2104             {
2105               vnet_ip_mcast_fixup_header (IP6_MCAST_ADDR_MASK,
2106                                           adj0->
2107                                           rewrite_header.dst_mcast_offset,
2108                                           &ip0->dst_address.as_u32[3],
2109                                           (u8 *) ip0);
2110             }
2111
2112           from += 1;
2113           n_left_from -= 1;
2114           to_next += 1;
2115           n_left_to_next -= 1;
2116
2117           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2118                                            to_next, n_left_to_next,
2119                                            pi0, next0);
2120         }
2121
2122       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2123     }
2124
2125   /* Need to do trace after rewrites to pick up new packet data. */
2126   if (node->flags & VLIB_NODE_FLAG_TRACE)
2127     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2128
2129   return frame->n_vectors;
2130 }
2131
2132 always_inline uword
2133 ip6_rewrite_inline (vlib_main_t * vm,
2134                     vlib_node_runtime_t * node,
2135                     vlib_frame_t * frame,
2136                     int do_counters, int is_midchain, int is_mcast)
2137 {
2138   return ip6_rewrite_inline_with_gso (vm, node, frame, do_counters,
2139                                       is_midchain, is_mcast);
2140 }
2141
2142 VLIB_NODE_FN (ip6_rewrite_node) (vlib_main_t * vm,
2143                                  vlib_node_runtime_t * node,
2144                                  vlib_frame_t * frame)
2145 {
2146   if (adj_are_counters_enabled ())
2147     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2148   else
2149     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2150 }
2151
2152 VLIB_NODE_FN (ip6_rewrite_bcast_node) (vlib_main_t * vm,
2153                                        vlib_node_runtime_t * node,
2154                                        vlib_frame_t * frame)
2155 {
2156   if (adj_are_counters_enabled ())
2157     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2158   else
2159     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2160 }
2161
2162 VLIB_NODE_FN (ip6_rewrite_mcast_node) (vlib_main_t * vm,
2163                                        vlib_node_runtime_t * node,
2164                                        vlib_frame_t * frame)
2165 {
2166   if (adj_are_counters_enabled ())
2167     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2168   else
2169     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2170 }
2171
2172 VLIB_NODE_FN (ip6_midchain_node) (vlib_main_t * vm,
2173                                   vlib_node_runtime_t * node,
2174                                   vlib_frame_t * frame)
2175 {
2176   if (adj_are_counters_enabled ())
2177     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2178   else
2179     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2180 }
2181
2182 VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
2183                                         vlib_node_runtime_t * node,
2184                                         vlib_frame_t * frame)
2185 {
2186   if (adj_are_counters_enabled ())
2187     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2188   else
2189     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2190 }
2191
2192 /* *INDENT-OFF* */
2193 VLIB_REGISTER_NODE (ip6_midchain_node) =
2194 {
2195   .name = "ip6-midchain",
2196   .vector_size = sizeof (u32),
2197   .format_trace = format_ip6_forward_next_trace,
2198   .sibling_of = "ip6-rewrite",
2199   };
2200
2201 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2202 {
2203   .name = "ip6-rewrite",
2204   .vector_size = sizeof (u32),
2205   .format_trace = format_ip6_rewrite_trace,
2206   .n_next_nodes = IP6_REWRITE_N_NEXT,
2207   .next_nodes =
2208   {
2209     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2210     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2211     [IP6_REWRITE_NEXT_FRAGMENT] = "ip6-frag",
2212   },
2213 };
2214
2215 VLIB_REGISTER_NODE (ip6_rewrite_bcast_node) = {
2216   .name = "ip6-rewrite-bcast",
2217   .vector_size = sizeof (u32),
2218
2219   .format_trace = format_ip6_rewrite_trace,
2220   .sibling_of = "ip6-rewrite",
2221 };
2222
2223 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2224 {
2225   .name = "ip6-rewrite-mcast",
2226   .vector_size = sizeof (u32),
2227   .format_trace = format_ip6_rewrite_trace,
2228   .sibling_of = "ip6-rewrite",
2229 };
2230
2231
2232 VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
2233 {
2234   .name = "ip6-mcast-midchain",
2235   .vector_size = sizeof (u32),
2236   .format_trace = format_ip6_rewrite_trace,
2237   .sibling_of = "ip6-rewrite",
2238 };
2239
2240 /* *INDENT-ON* */
2241
2242 /*
2243  * Hop-by-Hop handling
2244  */
2245 #ifndef CLIB_MARCH_VARIANT
2246 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2247 #endif /* CLIB_MARCH_VARIANT */
2248
2249 #define foreach_ip6_hop_by_hop_error \
2250 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2251 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2252 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2253
2254 /* *INDENT-OFF* */
2255 typedef enum
2256 {
2257 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2258   foreach_ip6_hop_by_hop_error
2259 #undef _
2260   IP6_HOP_BY_HOP_N_ERROR,
2261 } ip6_hop_by_hop_error_t;
2262 /* *INDENT-ON* */
2263
2264 /*
2265  * Primary h-b-h handler trace support
2266  * We work pretty hard on the problem for obvious reasons
2267  */
2268 typedef struct
2269 {
2270   u32 next_index;
2271   u32 trace_len;
2272   u8 option_data[256];
2273 } ip6_hop_by_hop_trace_t;
2274
2275 extern vlib_node_registration_t ip6_hop_by_hop_node;
2276
2277 static char *ip6_hop_by_hop_error_strings[] = {
2278 #define _(sym,string) string,
2279   foreach_ip6_hop_by_hop_error
2280 #undef _
2281 };
2282
2283 #ifndef CLIB_MARCH_VARIANT
2284 u8 *
2285 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2286 {
2287   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2288   int total_len = va_arg (*args, int);
2289   ip6_hop_by_hop_option_t *opt0, *limit0;
2290   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2291   u8 type0;
2292
2293   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2294               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2295
2296   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2297   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2298
2299   while (opt0 < limit0)
2300     {
2301       type0 = opt0->type;
2302       switch (type0)
2303         {
2304         case 0:         /* Pad, just stop */
2305           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2306           break;
2307
2308         default:
2309           if (hm->trace[type0])
2310             {
2311               s = (*hm->trace[type0]) (s, opt0);
2312             }
2313           else
2314             {
2315               s =
2316                 format (s, "\n    unrecognized option %d length %d", type0,
2317                         opt0->length);
2318             }
2319           opt0 =
2320             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2321                                          sizeof (ip6_hop_by_hop_option_t));
2322           break;
2323         }
2324     }
2325   return s;
2326 }
2327 #endif
2328
2329 static u8 *
2330 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2331 {
2332   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2333   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2334   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2335   ip6_hop_by_hop_header_t *hbh0;
2336   ip6_hop_by_hop_option_t *opt0, *limit0;
2337   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2338
2339   u8 type0;
2340
2341   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2342
2343   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2344               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2345
2346   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2347   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2348
2349   while (opt0 < limit0)
2350     {
2351       type0 = opt0->type;
2352       switch (type0)
2353         {
2354         case 0:         /* Pad, just stop */
2355           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2356           break;
2357
2358         default:
2359           if (hm->trace[type0])
2360             {
2361               s = (*hm->trace[type0]) (s, opt0);
2362             }
2363           else
2364             {
2365               s =
2366                 format (s, "\n    unrecognized option %d length %d", type0,
2367                         opt0->length);
2368             }
2369           opt0 =
2370             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2371                                          sizeof (ip6_hop_by_hop_option_t));
2372           break;
2373         }
2374     }
2375   return s;
2376 }
2377
2378 always_inline u8
2379 ip6_scan_hbh_options (vlib_buffer_t * b0,
2380                       ip6_header_t * ip0,
2381                       ip6_hop_by_hop_header_t * hbh0,
2382                       ip6_hop_by_hop_option_t * opt0,
2383                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2384 {
2385   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2386   u8 type0;
2387   u8 error0 = 0;
2388
2389   while (opt0 < limit0)
2390     {
2391       type0 = opt0->type;
2392       switch (type0)
2393         {
2394         case 0:         /* Pad1 */
2395           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2396           continue;
2397         case 1:         /* PadN */
2398           break;
2399         default:
2400           if (hm->options[type0])
2401             {
2402               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2403                 {
2404                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2405                   return (error0);
2406                 }
2407             }
2408           else
2409             {
2410               /* Unrecognized mandatory option, check the two high order bits */
2411               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2412                 {
2413                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2414                   break;
2415                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2416                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2417                   *next0 = IP_LOOKUP_NEXT_DROP;
2418                   break;
2419                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2420                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2421                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2422                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2423                                                ICMP6_parameter_problem_unrecognized_option,
2424                                                (u8 *) opt0 - (u8 *) ip0);
2425                   break;
2426                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2427                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2428                   if (!ip6_address_is_multicast (&ip0->dst_address))
2429                     {
2430                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2431                       icmp6_error_set_vnet_buffer (b0,
2432                                                    ICMP6_parameter_problem,
2433                                                    ICMP6_parameter_problem_unrecognized_option,
2434                                                    (u8 *) opt0 - (u8 *) ip0);
2435                     }
2436                   else
2437                     {
2438                       *next0 = IP_LOOKUP_NEXT_DROP;
2439                     }
2440                   break;
2441                 }
2442               return (error0);
2443             }
2444         }
2445       opt0 =
2446         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2447                                      sizeof (ip6_hop_by_hop_option_t));
2448     }
2449   return (error0);
2450 }
2451
2452 /*
2453  * Process the Hop-by-Hop Options header
2454  */
2455 VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
2456                                     vlib_node_runtime_t * node,
2457                                     vlib_frame_t * frame)
2458 {
2459   vlib_node_runtime_t *error_node =
2460     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2461   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2462   u32 n_left_from, *from, *to_next;
2463   ip_lookup_next_t next_index;
2464
2465   from = vlib_frame_vector_args (frame);
2466   n_left_from = frame->n_vectors;
2467   next_index = node->cached_next_index;
2468
2469   while (n_left_from > 0)
2470     {
2471       u32 n_left_to_next;
2472
2473       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2474
2475       while (n_left_from >= 4 && n_left_to_next >= 2)
2476         {
2477           u32 bi0, bi1;
2478           vlib_buffer_t *b0, *b1;
2479           u32 next0, next1;
2480           ip6_header_t *ip0, *ip1;
2481           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2482           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2483           u8 error0 = 0, error1 = 0;
2484
2485           /* Prefetch next iteration. */
2486           {
2487             vlib_buffer_t *p2, *p3;
2488
2489             p2 = vlib_get_buffer (vm, from[2]);
2490             p3 = vlib_get_buffer (vm, from[3]);
2491
2492             vlib_prefetch_buffer_header (p2, LOAD);
2493             vlib_prefetch_buffer_header (p3, LOAD);
2494
2495             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2496             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2497           }
2498
2499           /* Speculatively enqueue b0, b1 to the current next frame */
2500           to_next[0] = bi0 = from[0];
2501           to_next[1] = bi1 = from[1];
2502           from += 2;
2503           to_next += 2;
2504           n_left_from -= 2;
2505           n_left_to_next -= 2;
2506
2507           b0 = vlib_get_buffer (vm, bi0);
2508           b1 = vlib_get_buffer (vm, bi1);
2509
2510           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2511           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2512           ip_adjacency_t *adj0 = adj_get (adj_index0);
2513           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2514           ip_adjacency_t *adj1 = adj_get (adj_index1);
2515
2516           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2517           next0 = adj0->lookup_next_index;
2518           next1 = adj1->lookup_next_index;
2519
2520           ip0 = vlib_buffer_get_current (b0);
2521           ip1 = vlib_buffer_get_current (b1);
2522           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2523           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2524           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2525           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2526           limit0 =
2527             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2528                                          ((hbh0->length + 1) << 3));
2529           limit1 =
2530             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2531                                          ((hbh1->length + 1) << 3));
2532
2533           /*
2534            * Basic validity checks
2535            */
2536           if ((hbh0->length + 1) << 3 >
2537               clib_net_to_host_u16 (ip0->payload_length))
2538             {
2539               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2540               next0 = IP_LOOKUP_NEXT_DROP;
2541               goto outdual;
2542             }
2543           /* Scan the set of h-b-h options, process ones that we understand */
2544           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2545
2546           if ((hbh1->length + 1) << 3 >
2547               clib_net_to_host_u16 (ip1->payload_length))
2548             {
2549               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2550               next1 = IP_LOOKUP_NEXT_DROP;
2551               goto outdual;
2552             }
2553           /* Scan the set of h-b-h options, process ones that we understand */
2554           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2555
2556         outdual:
2557           /* Has the classifier flagged this buffer for special treatment? */
2558           if (PREDICT_FALSE
2559               ((error0 == 0)
2560                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2561             next0 = hm->next_override;
2562
2563           /* Has the classifier flagged this buffer for special treatment? */
2564           if (PREDICT_FALSE
2565               ((error1 == 0)
2566                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2567             next1 = hm->next_override;
2568
2569           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2570             {
2571               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2572                 {
2573                   ip6_hop_by_hop_trace_t *t =
2574                     vlib_add_trace (vm, node, b0, sizeof (*t));
2575                   u32 trace_len = (hbh0->length + 1) << 3;
2576                   t->next_index = next0;
2577                   /* Capture the h-b-h option verbatim */
2578                   trace_len =
2579                     trace_len <
2580                     ARRAY_LEN (t->option_data) ? trace_len :
2581                     ARRAY_LEN (t->option_data);
2582                   t->trace_len = trace_len;
2583                   clib_memcpy_fast (t->option_data, hbh0, trace_len);
2584                 }
2585               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2586                 {
2587                   ip6_hop_by_hop_trace_t *t =
2588                     vlib_add_trace (vm, node, b1, sizeof (*t));
2589                   u32 trace_len = (hbh1->length + 1) << 3;
2590                   t->next_index = next1;
2591                   /* Capture the h-b-h option verbatim */
2592                   trace_len =
2593                     trace_len <
2594                     ARRAY_LEN (t->option_data) ? trace_len :
2595                     ARRAY_LEN (t->option_data);
2596                   t->trace_len = trace_len;
2597                   clib_memcpy_fast (t->option_data, hbh1, trace_len);
2598                 }
2599
2600             }
2601
2602           b0->error = error_node->errors[error0];
2603           b1->error = error_node->errors[error1];
2604
2605           /* verify speculative enqueue, maybe switch current next frame */
2606           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2607                                            n_left_to_next, bi0, bi1, next0,
2608                                            next1);
2609         }
2610
2611       while (n_left_from > 0 && n_left_to_next > 0)
2612         {
2613           u32 bi0;
2614           vlib_buffer_t *b0;
2615           u32 next0;
2616           ip6_header_t *ip0;
2617           ip6_hop_by_hop_header_t *hbh0;
2618           ip6_hop_by_hop_option_t *opt0, *limit0;
2619           u8 error0 = 0;
2620
2621           /* Speculatively enqueue b0 to the current next frame */
2622           bi0 = from[0];
2623           to_next[0] = bi0;
2624           from += 1;
2625           to_next += 1;
2626           n_left_from -= 1;
2627           n_left_to_next -= 1;
2628
2629           b0 = vlib_get_buffer (vm, bi0);
2630           /*
2631            * Default use the next_index from the adjacency.
2632            * A HBH option rarely redirects to a different node
2633            */
2634           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2635           ip_adjacency_t *adj0 = adj_get (adj_index0);
2636           next0 = adj0->lookup_next_index;
2637
2638           ip0 = vlib_buffer_get_current (b0);
2639           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2640           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2641           limit0 =
2642             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2643                                          ((hbh0->length + 1) << 3));
2644
2645           /*
2646            * Basic validity checks
2647            */
2648           if ((hbh0->length + 1) << 3 >
2649               clib_net_to_host_u16 (ip0->payload_length))
2650             {
2651               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2652               next0 = IP_LOOKUP_NEXT_DROP;
2653               goto out0;
2654             }
2655
2656           /* Scan the set of h-b-h options, process ones that we understand */
2657           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2658
2659         out0:
2660           /* Has the classifier flagged this buffer for special treatment? */
2661           if (PREDICT_FALSE
2662               ((error0 == 0)
2663                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2664             next0 = hm->next_override;
2665
2666           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2667             {
2668               ip6_hop_by_hop_trace_t *t =
2669                 vlib_add_trace (vm, node, b0, sizeof (*t));
2670               u32 trace_len = (hbh0->length + 1) << 3;
2671               t->next_index = next0;
2672               /* Capture the h-b-h option verbatim */
2673               trace_len =
2674                 trace_len <
2675                 ARRAY_LEN (t->option_data) ? trace_len :
2676                 ARRAY_LEN (t->option_data);
2677               t->trace_len = trace_len;
2678               clib_memcpy_fast (t->option_data, hbh0, trace_len);
2679             }
2680
2681           b0->error = error_node->errors[error0];
2682
2683           /* verify speculative enqueue, maybe switch current next frame */
2684           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2685                                            n_left_to_next, bi0, next0);
2686         }
2687       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2688     }
2689   return frame->n_vectors;
2690 }
2691
2692 /* *INDENT-OFF* */
2693 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2694 {
2695   .name = "ip6-hop-by-hop",
2696   .sibling_of = "ip6-lookup",
2697   .vector_size = sizeof (u32),
2698   .format_trace = format_ip6_hop_by_hop_trace,
2699   .type = VLIB_NODE_TYPE_INTERNAL,
2700   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2701   .error_strings = ip6_hop_by_hop_error_strings,
2702   .n_next_nodes = 0,
2703 };
2704 /* *INDENT-ON* */
2705
2706 static clib_error_t *
2707 ip6_hop_by_hop_init (vlib_main_t * vm)
2708 {
2709   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2710   clib_memset (hm->options, 0, sizeof (hm->options));
2711   clib_memset (hm->trace, 0, sizeof (hm->trace));
2712   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2713   return (0);
2714 }
2715
2716 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2717
2718 #ifndef CLIB_MARCH_VARIANT
2719 void
2720 ip6_hbh_set_next_override (uword next)
2721 {
2722   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2723
2724   hm->next_override = next;
2725 }
2726
2727 int
2728 ip6_hbh_register_option (u8 option,
2729                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2730                                       ip6_hop_by_hop_option_t * opt),
2731                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2732 {
2733   ip6_main_t *im = &ip6_main;
2734   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2735
2736   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2737
2738   /* Already registered */
2739   if (hm->options[option])
2740     return (-1);
2741
2742   hm->options[option] = options;
2743   hm->trace[option] = trace;
2744
2745   /* Set global variable */
2746   im->hbh_enabled = 1;
2747
2748   return (0);
2749 }
2750
2751 int
2752 ip6_hbh_unregister_option (u8 option)
2753 {
2754   ip6_main_t *im = &ip6_main;
2755   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2756
2757   ASSERT ((u32) option < ARRAY_LEN (hm->options));
2758
2759   /* Not registered */
2760   if (!hm->options[option])
2761     return (-1);
2762
2763   hm->options[option] = NULL;
2764   hm->trace[option] = NULL;
2765
2766   /* Disable global knob if this was the last option configured */
2767   int i;
2768   bool found = false;
2769   for (i = 0; i < 256; i++)
2770     {
2771       if (hm->options[option])
2772         {
2773           found = true;
2774           break;
2775         }
2776     }
2777   if (!found)
2778     im->hbh_enabled = 0;
2779
2780   return (0);
2781 }
2782
2783 /* Global IP6 main. */
2784 ip6_main_t ip6_main;
2785 #endif
2786
2787 static clib_error_t *
2788 ip6_lookup_init (vlib_main_t * vm)
2789 {
2790   ip6_main_t *im = &ip6_main;
2791   clib_error_t *error;
2792   uword i;
2793
2794   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2795     return error;
2796
2797   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2798     {
2799       u32 j, i0, i1;
2800
2801       i0 = i / 32;
2802       i1 = i % 32;
2803
2804       for (j = 0; j < i0; j++)
2805         im->fib_masks[i].as_u32[j] = ~0;
2806
2807       if (i1)
2808         im->fib_masks[i].as_u32[i0] =
2809           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2810     }
2811
2812   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2813
2814   if (im->lookup_table_nbuckets == 0)
2815     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2816
2817   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2818
2819   if (im->lookup_table_size == 0)
2820     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2821
2822   clib_bihash_init_24_8 (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2823                          "ip6 FIB fwding table",
2824                          im->lookup_table_nbuckets, im->lookup_table_size);
2825   clib_bihash_init_24_8 (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2826                          "ip6 FIB non-fwding table",
2827                          im->lookup_table_nbuckets, im->lookup_table_size);
2828   clib_bihash_init_40_8 (&im->ip6_mtable.ip6_mhash,
2829                          "ip6 mFIB table",
2830                          im->lookup_table_nbuckets, im->lookup_table_size);
2831
2832   /* Create FIB with index 0 and table id of 0. */
2833   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2834                                      FIB_SOURCE_DEFAULT_ROUTE);
2835   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2836                                       MFIB_SOURCE_DEFAULT_ROUTE);
2837
2838   {
2839     pg_node_t *pn;
2840     pn = pg_get_node (ip6_lookup_node.index);
2841     pn->unformat_edit = unformat_pg_ip6_header;
2842   }
2843
2844   /* Unless explicitly configured, don't process HBH options */
2845   im->hbh_enabled = 0;
2846
2847   return error;
2848 }
2849
2850 VLIB_INIT_FUNCTION (ip6_lookup_init);
2851
2852 #ifndef CLIB_MARCH_VARIANT
2853 int
2854 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2855 {
2856   u32 fib_index;
2857
2858   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2859
2860   if (~0 == fib_index)
2861     return VNET_API_ERROR_NO_SUCH_FIB;
2862
2863   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2864                                   flow_hash_config);
2865
2866   return 0;
2867 }
2868 #endif
2869
2870 static clib_error_t *
2871 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2872                               unformat_input_t * input,
2873                               vlib_cli_command_t * cmd)
2874 {
2875   int matched = 0;
2876   u32 table_id = 0;
2877   u32 flow_hash_config = 0;
2878   int rv;
2879
2880   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2881     {
2882       if (unformat (input, "table %d", &table_id))
2883         matched = 1;
2884 #define _(a,v) \
2885     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2886       foreach_flow_hash_bit
2887 #undef _
2888         else
2889         break;
2890     }
2891
2892   if (matched == 0)
2893     return clib_error_return (0, "unknown input `%U'",
2894                               format_unformat_error, input);
2895
2896   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2897   switch (rv)
2898     {
2899     case 0:
2900       break;
2901
2902     case -1:
2903       return clib_error_return (0, "no such FIB table %d", table_id);
2904
2905     default:
2906       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2907       break;
2908     }
2909
2910   return 0;
2911 }
2912
2913 /*?
2914  * Configure the set of IPv6 fields used by the flow hash.
2915  *
2916  * @cliexpar
2917  * @parblock
2918  * Example of how to set the flow hash on a given table:
2919  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2920  *
2921  * Example of display the configured flow hash:
2922  * @cliexstart{show ip6 fib}
2923  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2924  * @::/0
2925  *   unicast-ip6-chain
2926  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2927  *     [0] [@0]: dpo-drop ip6
2928  * fe80::/10
2929  *   unicast-ip6-chain
2930  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2931  *     [0] [@2]: dpo-receive
2932  * ff02::1/128
2933  *   unicast-ip6-chain
2934  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2935  *     [0] [@2]: dpo-receive
2936  * ff02::2/128
2937  *   unicast-ip6-chain
2938  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2939  *     [0] [@2]: dpo-receive
2940  * ff02::16/128
2941  *   unicast-ip6-chain
2942  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2943  *     [0] [@2]: dpo-receive
2944  * ff02::1:ff00:0/104
2945  *   unicast-ip6-chain
2946  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2947  *     [0] [@2]: dpo-receive
2948  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2949  * @::/0
2950  *   unicast-ip6-chain
2951  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2952  *     [0] [@0]: dpo-drop ip6
2953  * @::a:1:1:0:4/126
2954  *   unicast-ip6-chain
2955  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2956  *     [0] [@4]: ipv6-glean: af_packet0
2957  * @::a:1:1:0:7/128
2958  *   unicast-ip6-chain
2959  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2960  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2961  * fe80::/10
2962  *   unicast-ip6-chain
2963  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2964  *     [0] [@2]: dpo-receive
2965  * fe80::fe:3eff:fe3e:9222/128
2966  *   unicast-ip6-chain
2967  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2968  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2969  * ff02::1/128
2970  *   unicast-ip6-chain
2971  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2972  *     [0] [@2]: dpo-receive
2973  * ff02::2/128
2974  *   unicast-ip6-chain
2975  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2976  *     [0] [@2]: dpo-receive
2977  * ff02::16/128
2978  *   unicast-ip6-chain
2979  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2980  *     [0] [@2]: dpo-receive
2981  * ff02::1:ff00:0/104
2982  *   unicast-ip6-chain
2983  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2984  *     [0] [@2]: dpo-receive
2985  * @cliexend
2986  * @endparblock
2987 ?*/
2988 /* *INDENT-OFF* */
2989 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
2990 {
2991   .path = "set ip6 flow-hash",
2992   .short_help =
2993   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2994   .function = set_ip6_flow_hash_command_fn,
2995 };
2996 /* *INDENT-ON* */
2997
2998 static clib_error_t *
2999 show_ip6_local_command_fn (vlib_main_t * vm,
3000                            unformat_input_t * input, vlib_cli_command_t * cmd)
3001 {
3002   ip6_main_t *im = &ip6_main;
3003   ip_lookup_main_t *lm = &im->lookup_main;
3004   int i;
3005
3006   vlib_cli_output (vm, "Protocols handled by ip6_local");
3007   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
3008     {
3009       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3010         {
3011
3012           u32 node_index = vlib_get_node (vm,
3013                                           ip6_local_node.index)->
3014             next_nodes[lm->local_next_by_ip_protocol[i]];
3015           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
3016                            node_index);
3017         }
3018     }
3019   return 0;
3020 }
3021
3022
3023
3024 /*?
3025  * Display the set of protocols handled by the local IPv6 stack.
3026  *
3027  * @cliexpar
3028  * Example of how to display local protocol table:
3029  * @cliexstart{show ip6 local}
3030  * Protocols handled by ip6_local
3031  * 17
3032  * 43
3033  * 58
3034  * 115
3035  * @cliexend
3036 ?*/
3037 /* *INDENT-OFF* */
3038 VLIB_CLI_COMMAND (show_ip6_local, static) =
3039 {
3040   .path = "show ip6 local",
3041   .function = show_ip6_local_command_fn,
3042   .short_help = "show ip6 local",
3043 };
3044 /* *INDENT-ON* */
3045
3046 #ifndef CLIB_MARCH_VARIANT
3047 int
3048 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3049                              u32 table_index)
3050 {
3051   vnet_main_t *vnm = vnet_get_main ();
3052   vnet_interface_main_t *im = &vnm->interface_main;
3053   ip6_main_t *ipm = &ip6_main;
3054   ip_lookup_main_t *lm = &ipm->lookup_main;
3055   vnet_classify_main_t *cm = &vnet_classify_main;
3056   ip6_address_t *if_addr;
3057
3058   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3059     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3060
3061   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3062     return VNET_API_ERROR_NO_SUCH_ENTRY;
3063
3064   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3065   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3066
3067   if_addr = ip6_interface_first_address (ipm, sw_if_index);
3068
3069   if (NULL != if_addr)
3070     {
3071       fib_prefix_t pfx = {
3072         .fp_len = 128,
3073         .fp_proto = FIB_PROTOCOL_IP6,
3074         .fp_addr.ip6 = *if_addr,
3075       };
3076       u32 fib_index;
3077
3078       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3079                                                        sw_if_index);
3080
3081
3082       if (table_index != (u32) ~ 0)
3083         {
3084           dpo_id_t dpo = DPO_INVALID;
3085
3086           dpo_set (&dpo,
3087                    DPO_CLASSIFY,
3088                    DPO_PROTO_IP6,
3089                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3090
3091           fib_table_entry_special_dpo_add (fib_index,
3092                                            &pfx,
3093                                            FIB_SOURCE_CLASSIFY,
3094                                            FIB_ENTRY_FLAG_NONE, &dpo);
3095           dpo_reset (&dpo);
3096         }
3097       else
3098         {
3099           fib_table_entry_special_remove (fib_index,
3100                                           &pfx, FIB_SOURCE_CLASSIFY);
3101         }
3102     }
3103
3104   return 0;
3105 }
3106 #endif
3107
3108 static clib_error_t *
3109 set_ip6_classify_command_fn (vlib_main_t * vm,
3110                              unformat_input_t * input,
3111                              vlib_cli_command_t * cmd)
3112 {
3113   u32 table_index = ~0;
3114   int table_index_set = 0;
3115   u32 sw_if_index = ~0;
3116   int rv;
3117
3118   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3119     {
3120       if (unformat (input, "table-index %d", &table_index))
3121         table_index_set = 1;
3122       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3123                          vnet_get_main (), &sw_if_index))
3124         ;
3125       else
3126         break;
3127     }
3128
3129   if (table_index_set == 0)
3130     return clib_error_return (0, "classify table-index must be specified");
3131
3132   if (sw_if_index == ~0)
3133     return clib_error_return (0, "interface / subif must be specified");
3134
3135   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3136
3137   switch (rv)
3138     {
3139     case 0:
3140       break;
3141
3142     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3143       return clib_error_return (0, "No such interface");
3144
3145     case VNET_API_ERROR_NO_SUCH_ENTRY:
3146       return clib_error_return (0, "No such classifier table");
3147     }
3148   return 0;
3149 }
3150
3151 /*?
3152  * Assign a classification table to an interface. The classification
3153  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3154  * commands. Once the table is create, use this command to filter packets
3155  * on an interface.
3156  *
3157  * @cliexpar
3158  * Example of how to assign a classification table to an interface:
3159  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3160 ?*/
3161 /* *INDENT-OFF* */
3162 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3163 {
3164   .path = "set ip6 classify",
3165   .short_help =
3166   "set ip6 classify intfc <interface> table-index <classify-idx>",
3167   .function = set_ip6_classify_command_fn,
3168 };
3169 /* *INDENT-ON* */
3170
3171 static clib_error_t *
3172 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3173 {
3174   ip6_main_t *im = &ip6_main;
3175   uword heapsize = 0;
3176   u32 tmp;
3177   u32 nbuckets = 0;
3178
3179   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3180     {
3181       if (unformat (input, "hash-buckets %d", &tmp))
3182         nbuckets = tmp;
3183       else if (unformat (input, "heap-size %U",
3184                          unformat_memory_size, &heapsize))
3185         ;
3186       else
3187         return clib_error_return (0, "unknown input '%U'",
3188                                   format_unformat_error, input);
3189     }
3190
3191   im->lookup_table_nbuckets = nbuckets;
3192   im->lookup_table_size = heapsize;
3193
3194   return 0;
3195 }
3196
3197 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3198
3199 /*
3200  * fd.io coding-style-patch-verification: ON
3201  *
3202  * Local Variables:
3203  * eval: (c-set-style "gnu")
3204  * End:
3205  */