vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45 #include <vnet/fib/fib_table.h>
  46 #include <vnet/fib/ip6_fib.h>
  47 #include <vnet/dpo/load_balance.h>
  48 #include <vnet/dpo/classify_dpo.h>
  49
  50 #include <vppinfra/bihash_template.c>
  51
  52 /**
  53  * @file
  54  * @brief IPv6 Forwarding.
  55  *
  56  * This file contains the source code for IPv6 forwarding.
  57  */
  58
  59 void
  60 ip6_forward_next_trace (vlib_main_t * vm,
  61                         vlib_node_runtime_t * node,
  62                         vlib_frame_t * frame,
  63                         vlib_rx_or_tx_t which_adj_index);
  64
  65 always_inline uword
  66 ip6_lookup_inline (vlib_main_t * vm,
  67                    vlib_node_runtime_t * node,
  68                    vlib_frame_t * frame)
  69 {
  70   ip6_main_t * im = &ip6_main;
  71   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  72   u32 n_left_from, n_left_to_next, * from, * to_next;
  73   ip_lookup_next_t next;
  74   u32 cpu_index = os_get_cpu_number();
  75
  76   from = vlib_frame_vector_args (frame);
  77   n_left_from = frame->n_vectors;
  78   next = node->cached_next_index;
  79
  80   while (n_left_from > 0)
  81     {
  82       vlib_get_next_frame (vm, node, next,
  83                            to_next, n_left_to_next);
  84
  85       while (n_left_from >= 4 && n_left_to_next >= 2)
  86         {
  87           vlib_buffer_t * p0, * p1;
  88           u32 pi0, pi1, lbi0, lbi1, wrong_next;
  89           ip_lookup_next_t next0, next1;
  90           ip6_header_t * ip0, * ip1;
  91           ip6_address_t * dst_addr0, * dst_addr1;
  92           u32 fib_index0, fib_index1;
  93           u32 flow_hash_config0, flow_hash_config1;
  94           const dpo_id_t *dpo0, *dpo1;
  95           const load_balance_t *lb0, *lb1;
  96
  97           /* Prefetch next iteration. */
  98           {
  99             vlib_buffer_t * p2, * p3;
 100
 101             p2 = vlib_get_buffer (vm, from[2]);
 102             p3 = vlib_get_buffer (vm, from[3]);
 103
 104             vlib_prefetch_buffer_header (p2, LOAD);
 105             vlib_prefetch_buffer_header (p3, LOAD);
 106             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 107             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 108           }
 109
 110           pi0 = to_next[0] = from[0];
 111           pi1 = to_next[1] = from[1];
 112
 113           p0 = vlib_get_buffer (vm, pi0);
 114           p1 = vlib_get_buffer (vm, pi1);
 115
 116           ip0 = vlib_buffer_get_current (p0);
 117           ip1 = vlib_buffer_get_current (p1);
 118
 119           dst_addr0 = &ip0->dst_address;
 120           dst_addr1 = &ip1->dst_address;
 121
 122           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 123           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 124
 125           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 126             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 127           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 128             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 129
 130           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 131           lbi1 = ip6_fib_table_fwding_lookup (im, fib_index1, dst_addr1);
 132
 133           lb0 = load_balance_get (lbi0);
 134           lb1 = load_balance_get (lbi1);
 135
 136           vnet_buffer (p0)->ip.flow_hash =
 137             vnet_buffer(p1)->ip.flow_hash = 0;
 138
 139           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 140             {
 141               flow_hash_config0 = lb0->lb_hash_config;
 142               vnet_buffer (p0)->ip.flow_hash =
 143                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 144             }
 145           if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
 146             {
 147               flow_hash_config1 = lb1->lb_hash_config;
 148               vnet_buffer (p1)->ip.flow_hash =
 149                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 150             }
 151
 152           ASSERT (lb0->lb_n_buckets > 0);
 153           ASSERT (lb1->lb_n_buckets > 0);
 154           ASSERT (is_pow2 (lb0->lb_n_buckets));
 155           ASSERT (is_pow2 (lb1->lb_n_buckets));
 156           dpo0 = load_balance_get_bucket_i(lb0,
 157                                            (vnet_buffer (p0)->ip.flow_hash &
 158                                             lb0->lb_n_buckets_minus_1));
 159           dpo1 = load_balance_get_bucket_i(lb1,
 160                                            (vnet_buffer (p1)->ip.flow_hash &
 161                                             lb1->lb_n_buckets_minus_1));
 162
 163           next0 = dpo0->dpoi_next_node;
 164           next1 = dpo1->dpoi_next_node;
 165
 166           /* Only process the HBH Option Header if explicitly configured to do so */
 167           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 168             {
 169               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 170                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 171             }
 172           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 173             {
 174               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 175                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 176             }
 177           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 178           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 179
 180           vlib_increment_combined_counter
 181               (cm, cpu_index, lbi0, 1,
 182                vlib_buffer_length_in_chain (vm, p0));
 183           vlib_increment_combined_counter
 184               (cm, cpu_index, lbi1, 1,
 185                vlib_buffer_length_in_chain (vm, p1));
 186
 187           from += 2;
 188           to_next += 2;
 189           n_left_to_next -= 2;
 190           n_left_from -= 2;
 191
 192           wrong_next = (next0 != next) + 2*(next1 != next);
 193           if (PREDICT_FALSE (wrong_next != 0))
 194             {
 195               switch (wrong_next)
 196                 {
 197                 case 1:
 198                   /* A B A */
 199                   to_next[-2] = pi1;
 200                   to_next -= 1;
 201                   n_left_to_next += 1;
 202                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 203                   break;
 204
 205                 case 2:
 206                   /* A A B */
 207                   to_next -= 1;
 208                   n_left_to_next += 1;
 209                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 210                   break;
 211
 212                 case 3:
 213                   /* A B C */
 214                   to_next -= 2;
 215                   n_left_to_next += 2;
 216                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 217                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 218                   if (next0 == next1)
 219                     {
 220                       /* A B B */
 221                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 222                       next = next1;
 223                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 224                     }
 225                 }
 226             }
 227         }
 228
 229       while (n_left_from > 0 && n_left_to_next > 0)
 230         {
 231           vlib_buffer_t * p0;
 232           ip6_header_t * ip0;
 233           u32 pi0, lbi0;
 234           ip_lookup_next_t next0;
 235           load_balance_t * lb0;
 236           ip6_address_t * dst_addr0;
 237           u32 fib_index0, flow_hash_config0;
 238           const dpo_id_t *dpo0;
 239
 240           pi0 = from[0];
 241           to_next[0] = pi0;
 242
 243           p0 = vlib_get_buffer (vm, pi0);
 244
 245           ip0 = vlib_buffer_get_current (p0);
 246
 247           dst_addr0 = &ip0->dst_address;
 248
 249           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 250           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 251             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 252
 253           flow_hash_config0 =
 254               ip6_fib_get (fib_index0)->flow_hash_config;
 255
 256           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 257
 258           lb0 = load_balance_get (lbi0);
 259
 260           vnet_buffer (p0)->ip.flow_hash = 0;
 261
 262           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 263             {
 264               flow_hash_config0 = lb0->lb_hash_config;
 265               vnet_buffer (p0)->ip.flow_hash =
 266                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 267             }
 268
 269           ASSERT (lb0->lb_n_buckets > 0);
 270           ASSERT (is_pow2 (lb0->lb_n_buckets));
 271           dpo0 = load_balance_get_bucket_i(lb0,
 272                                            (vnet_buffer (p0)->ip.flow_hash &
 273                                             lb0->lb_n_buckets_minus_1));
 274           next0 = dpo0->dpoi_next_node;
 275
 276           /* Only process the HBH Option Header if explicitly configured to do so */
 277           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 278             {
 279               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 280                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 281             }
 282           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 283
 284           vlib_increment_combined_counter
 285               (cm, cpu_index, lbi0, 1,
 286                vlib_buffer_length_in_chain (vm, p0));
 287
 288           from += 1;
 289           to_next += 1;
 290           n_left_to_next -= 1;
 291           n_left_from -= 1;
 292
 293           if (PREDICT_FALSE (next0 != next))
 294             {
 295               n_left_to_next += 1;
 296               vlib_put_next_frame (vm, node, next, n_left_to_next);
 297               next = next0;
 298               vlib_get_next_frame (vm, node, next,
 299                                    to_next, n_left_to_next);
 300               to_next[0] = pi0;
 301               to_next += 1;
 302               n_left_to_next -= 1;
 303             }
 304         }
 305
 306       vlib_put_next_frame (vm, node, next, n_left_to_next);
 307     }
 308
 309   if (node->flags & VLIB_NODE_FLAG_TRACE)
 310     ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 311
 312   return frame->n_vectors;
 313 }
 314
 315 static void
 316 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 317                           ip6_main_t * im, u32 fib_index,
 318                           ip_interface_address_t * a)
 319 {
 320   ip_lookup_main_t * lm = &im->lookup_main;
 321   ip6_address_t * address = ip_interface_address_get_address (lm, a);
 322   fib_prefix_t pfx = {
 323       .fp_len = a->address_length,
 324       .fp_proto = FIB_PROTOCOL_IP6,
 325       .fp_addr.ip6 = *address,
 326   };
 327
 328   a->neighbor_probe_adj_index = ~0;
 329   if (a->address_length < 128)
 330   {
 331       fib_node_index_t fei;
 332
 333       fei = fib_table_entry_update_one_path(fib_index,
 334                                             &pfx,
 335                                             FIB_SOURCE_INTERFACE,
 336                                             (FIB_ENTRY_FLAG_CONNECTED |
 337                                              FIB_ENTRY_FLAG_ATTACHED),
 338                                             FIB_PROTOCOL_IP6,
 339                                             NULL, /* No next-hop address */
 340                                             sw_if_index,
 341                                             ~0, // invalid FIB index
 342                                             1,
 343                                             NULL, // no label stack
 344                                             FIB_ROUTE_PATH_FLAG_NONE);
 345       a->neighbor_probe_adj_index = fib_entry_get_adj(fei);
 346   }
 347
 348   pfx.fp_len = 128;
 349   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 350   {
 351       u32 classify_table_index =
 352           lm->classify_table_index_by_sw_if_index [sw_if_index];
 353       if (classify_table_index != (u32) ~0)
 354       {
 355           dpo_id_t dpo = DPO_INVALID;
 356
 357           dpo_set(&dpo,
 358                   DPO_CLASSIFY,
 359                   DPO_PROTO_IP6,
 360                   classify_dpo_create(DPO_PROTO_IP6, classify_table_index));
 361
 362           fib_table_entry_special_dpo_add(fib_index,
 363                                           &pfx,
 364                                           FIB_SOURCE_CLASSIFY,
 365                                           FIB_ENTRY_FLAG_NONE,
 366                                           &dpo);
 367           dpo_reset(&dpo);
 368       }
 369   }
 370
 371   fib_table_entry_update_one_path(fib_index,
 372                                   &pfx,
 373                                   FIB_SOURCE_INTERFACE,
 374                                   (FIB_ENTRY_FLAG_CONNECTED |
 375                                    FIB_ENTRY_FLAG_LOCAL),
 376                                   FIB_PROTOCOL_IP6,
 377                                   &pfx.fp_addr,
 378                                   sw_if_index,
 379                                   ~0, // invalid FIB index
 380                                   1,
 381                                   NULL,
 382                                   FIB_ROUTE_PATH_FLAG_NONE);
 383 }
 384
 385 static void
 386 ip6_del_interface_routes (ip6_main_t * im,
 387                           u32 fib_index,
 388                           ip6_address_t * address,
 389                           u32 address_length)
 390 {
 391     fib_prefix_t pfx = {
 392         .fp_len = address_length,
 393         .fp_proto = FIB_PROTOCOL_IP6,
 394         .fp_addr.ip6 = *address,
 395     };
 396
 397     if (pfx.fp_len < 128)
 398     {
 399         fib_table_entry_delete(fib_index,
 400                                &pfx,
 401                                FIB_SOURCE_INTERFACE);
 402
 403     }
 404
 405     pfx.fp_len = 128;
 406     fib_table_entry_delete(fib_index,
 407                            &pfx,
 408                            FIB_SOURCE_INTERFACE);
 409 }
 410
 411 void
 412 ip6_sw_interface_enable_disable (u32 sw_if_index,
 413                                  u32 is_enable)
 414 {
 415   ip6_main_t * im = &ip6_main;
 416
 417   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 418
 419   /*
 420    * enable/disable only on the 1<->0 transition
 421    */
 422   if (is_enable)
 423     {
 424       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 425         return;
 426     }
 427   else
 428     {
 429       ASSERT(im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 430       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 431         return;
 432     }
 433
 434   vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index,
 435                                is_enable, 0, 0);
 436
 437   vnet_feature_enable_disable ("ip6-multicast", "ip6-lookup", sw_if_index,
 438                                is_enable, 0, 0);
 439
 440 }
 441
 442 /* get first interface address */
 443 ip6_address_t *
 444 ip6_interface_first_address (ip6_main_t * im,
 445                              u32 sw_if_index,
 446                              ip_interface_address_t ** result_ia)
 447 {
 448   ip_lookup_main_t * lm = &im->lookup_main;
 449   ip_interface_address_t * ia = 0;
 450   ip6_address_t * result = 0;
 451
 452   foreach_ip_interface_address (lm, ia, sw_if_index,
 453                                 1 /* honor unnumbered */,
 454   ({
 455     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 456     result = a;
 457     break;
 458   }));
 459   if (result_ia)
 460     *result_ia = result ? ia : 0;
 461   return result;
 462 }
 463
 464 clib_error_t *
 465 ip6_add_del_interface_address (vlib_main_t * vm,
 466                                u32 sw_if_index,
 467                                ip6_address_t * address,
 468                                u32 address_length,
 469                                u32 is_del)
 470 {
 471   vnet_main_t * vnm = vnet_get_main();
 472   ip6_main_t * im = &ip6_main;
 473   ip_lookup_main_t * lm = &im->lookup_main;
 474   clib_error_t * error;
 475   u32 if_address_index;
 476   ip6_address_fib_t ip6_af, * addr_fib = 0;
 477
 478   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 479   ip6_addr_fib_init (&ip6_af, address,
 480                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 481   vec_add1 (addr_fib, ip6_af);
 482
 483   {
 484     uword elts_before = pool_elts (lm->if_address_pool);
 485
 486     error = ip_interface_address_add_del
 487       (lm,
 488        sw_if_index,
 489        addr_fib,
 490        address_length,
 491        is_del,
 492        &if_address_index);
 493     if (error)
 494       goto done;
 495
 496     /* Pool did not grow: add duplicate address. */
 497     if (elts_before == pool_elts (lm->if_address_pool))
 498       goto done;
 499   }
 500
 501   ip6_sw_interface_enable_disable(sw_if_index, !is_del);
 502
 503   if (is_del)
 504       ip6_del_interface_routes (im, ip6_af.fib_index, address,
 505                                 address_length);
 506   else
 507       ip6_add_interface_routes (vnm, sw_if_index,
 508                                 im, ip6_af.fib_index,
 509                                 pool_elt_at_index (lm->if_address_pool, if_address_index));
 510
 511   {
 512     ip6_add_del_interface_address_callback_t * cb;
 513     vec_foreach (cb, im->add_del_interface_address_callbacks)
 514       cb->function (im, cb->function_opaque, sw_if_index,
 515                     address, address_length,
 516                     if_address_index,
 517                     is_del);
 518   }
 519
 520  done:
 521   vec_free (addr_fib);
 522   return error;
 523 }
 524
 525 clib_error_t *
 526 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
 527                                 u32 sw_if_index,
 528                                 u32 flags)
 529 {
 530   ip6_main_t * im = &ip6_main;
 531   ip_interface_address_t * ia;
 532   ip6_address_t * a;
 533   u32 is_admin_up, fib_index;
 534
 535   /* Fill in lookup tables with default table (0). */
 536   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 537
 538   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
 539
 540   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 541
 542   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 543
 544   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 545                                 0 /* honor unnumbered */,
 546   ({
 547     a = ip_interface_address_get_address (&im->lookup_main, ia);
 548     if (is_admin_up)
 549       ip6_add_interface_routes (vnm, sw_if_index,
 550                                 im, fib_index,
 551                                 ia);
 552     else
 553       ip6_del_interface_routes (im, fib_index,
 554                                 a, ia->address_length);
 555   }));
 556
 557   return 0;
 558 }
 559
 560 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 561
 562 /* Built-in ip6 unicast rx feature path definition */
 563 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 564 {
 565   .arc_name  = "ip6-unicast",
 566   .start_nodes = VNET_FEATURES ("ip6-input"),
 567   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 568 };
 569
 570 VNET_FEATURE_INIT (ip6_flow_classify, static) = {
 571   .arc_name = "ip6-unicast",
 572   .node_name = "ip6-flow-classify",
 573   .runs_before = VNET_FEATURES ("ip6-inacl"),
 574 };
 575
 576 VNET_FEATURE_INIT (ip6_inacl, static) = {
 577   .arc_name = "ip6-unicast",
 578   .node_name = "ip6-inacl",
 579   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 580 };
 581
 582 VNET_FEATURE_INIT (ip6_policer_classify, static) = {
 583   .arc_name = "ip6-unicast",
 584   .node_name = "ip6-policer-classify",
 585   .runs_before = VNET_FEATURES ("ipsec-input-ip6"),
 586 };
 587
 588 VNET_FEATURE_INIT (ip6_ipsec, static) = {
 589   .arc_name = "ip6-unicast",
 590   .node_name = "ipsec-input-ip6",
 591   .runs_before = VNET_FEATURES ("l2tp-decap"),
 592 };
 593
 594 VNET_FEATURE_INIT (ip6_l2tp, static) = {
 595   .arc_name = "ip6-unicast",
 596   .node_name = "l2tp-decap",
 597   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 598 };
 599
 600 VNET_FEATURE_INIT (ip6_vpath, static) = {
 601   .arc_name = "ip6-unicast",
 602   .node_name = "vpath-input-ip6",
 603   .runs_before = VNET_FEATURES ("ip6-lookup"),
 604 };
 605
 606 VNET_FEATURE_INIT (ip6_lookup, static) = {
 607   .arc_name = "ip6-unicast",
 608   .node_name = "ip6-lookup",
 609   .runs_before = VNET_FEATURES ("ip6-drop"),
 610 };
 611
 612 VNET_FEATURE_INIT (ip6_drop, static) = {
 613   .arc_name = "ip6-unicast",
 614   .node_name = "ip6-drop",
 615   .runs_before = 0,  /*last feature*/
 616 };
 617
 618 /* Built-in ip6 multicast rx feature path definition (none now) */
 619 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 620 {
 621   .arc_name  = "ip6-multicast",
 622   .start_nodes = VNET_FEATURES ("ip6-input"),
 623   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 624 };
 625
 626 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 627   .arc_name = "ip6-multicast",
 628   .node_name = "vpath-input-ip6",
 629   .runs_before = VNET_FEATURES ("ip6-lookup"),
 630 };
 631
 632 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 633   .arc_name = "ip6-multicast",
 634   .node_name = "ip6-lookup",
 635   .runs_before = VNET_FEATURES ("ip6-drop"),
 636 };
 637
 638 VNET_FEATURE_INIT (ip6_drop_mc, static) = {
 639   .arc_name = "ip6-multicast",
 640   .node_name = "ip6-drop",
 641   .runs_before = 0, /* last feature */
 642 };
 643
 644 /* Built-in ip4 tx feature path definition */
 645 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 646 {
 647   .arc_name  = "ip6-output",
 648   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain"),
 649   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 650 };
 651
 652 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 653   .arc_name = "ip6-output",
 654   .node_name = "ipsec-output-ip6",
 655   .runs_before = VNET_FEATURES ("interface-output"),
 656 };
 657
 658 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 659   .arc_name = "ip6-output",
 660   .node_name = "interface-output",
 661   .runs_before = 0, /* not before any other features */
 662 };
 663
 664 clib_error_t *
 665 ip6_sw_interface_add_del (vnet_main_t * vnm,
 666                           u32 sw_if_index,
 667                           u32 is_add)
 668 {
 669   vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index,
 670                                is_add, 0, 0);
 671
 672   vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index,
 673                                is_add, 0, 0);
 674
 675   vnet_feature_enable_disable ("ip6-output", "interface-output", sw_if_index,
 676                                is_add, 0, 0);
 677
 678   return /* no error */ 0;
 679 }
 680
 681 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 682
 683 static uword
 684 ip6_lookup (vlib_main_t * vm,
 685             vlib_node_runtime_t * node,
 686             vlib_frame_t * frame)
 687 {
 688   return ip6_lookup_inline (vm, node, frame);
 689 }
 690
 691 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
 692
 693 VLIB_REGISTER_NODE (ip6_lookup_node) = {
 694   .function = ip6_lookup,
 695   .name = "ip6-lookup",
 696   .vector_size = sizeof (u32),
 697
 698   .format_trace = format_ip6_lookup_trace,
 699
 700   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 701   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 702 };
 703
 704 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
 705
 706 always_inline uword
 707 ip6_load_balance (vlib_main_t * vm,
 708                   vlib_node_runtime_t * node,
 709                   vlib_frame_t * frame)
 710 {
 711   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
 712   u32 n_left_from, n_left_to_next, * from, * to_next;
 713   ip_lookup_next_t next;
 714   u32 cpu_index = os_get_cpu_number();
 715   ip6_main_t * im = &ip6_main;
 716
 717   from = vlib_frame_vector_args (frame);
 718   n_left_from = frame->n_vectors;
 719   next = node->cached_next_index;
 720
 721   if (node->flags & VLIB_NODE_FLAG_TRACE)
 722       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 723
 724   while (n_left_from > 0)
 725     {
 726       vlib_get_next_frame (vm, node, next,
 727                            to_next, n_left_to_next);
 728
 729
 730       while (n_left_from >= 4 && n_left_to_next >= 2)
 731         {
 732           ip_lookup_next_t next0, next1;
 733           const load_balance_t *lb0, *lb1;
 734           vlib_buffer_t * p0, *p1;
 735           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 736           const ip6_header_t *ip0, *ip1;
 737           const dpo_id_t *dpo0, *dpo1;
 738
 739           /* Prefetch next iteration. */
 740           {
 741             vlib_buffer_t * p2, * p3;
 742
 743             p2 = vlib_get_buffer (vm, from[2]);
 744             p3 = vlib_get_buffer (vm, from[3]);
 745
 746             vlib_prefetch_buffer_header (p2, STORE);
 747             vlib_prefetch_buffer_header (p3, STORE);
 748
 749             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 750             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 751           }
 752
 753           pi0 = to_next[0] = from[0];
 754           pi1 = to_next[1] = from[1];
 755
 756           from += 2;
 757           n_left_from -= 2;
 758           to_next += 2;
 759           n_left_to_next -= 2;
 760
 761           p0 = vlib_get_buffer (vm, pi0);
 762           p1 = vlib_get_buffer (vm, pi1);
 763
 764           ip0 = vlib_buffer_get_current (p0);
 765           ip1 = vlib_buffer_get_current (p1);
 766           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 767           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 768
 769           lb0 = load_balance_get(lbi0);
 770           lb1 = load_balance_get(lbi1);
 771
 772           /*
 773            * this node is for via FIBs we can re-use the hash value from the
 774            * to node if present.
 775            * We don't want to use the same hash value at each level in the recursion
 776            * graph as that would lead to polarisation
 777            */
 778           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 779           hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
 780
 781           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 782           {
 783               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 784               {
 785                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 786               }
 787               else
 788               {
 789                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 790               }
 791           }
 792           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 793           {
 794               if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash))
 795               {
 796                   hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1;
 797               }
 798               else
 799               {
 800                   hc1 = vnet_buffer(p1)->ip.flow_hash = ip6_compute_flow_hash(ip1, hc1);
 801               }
 802           }
 803
 804           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 805           dpo1 = load_balance_get_bucket_i(lb1, hc1 & (lb1->lb_n_buckets_minus_1));
 806
 807           next0 = dpo0->dpoi_next_node;
 808           next1 = dpo1->dpoi_next_node;
 809
 810           /* Only process the HBH Option Header if explicitly configured to do so */
 811           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 812             {
 813               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 814                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 815             }
 816           /* Only process the HBH Option Header if explicitly configured to do so */
 817           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 818             {
 819               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 820                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 821             }
 822
 823           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 824           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 825
 826           vlib_increment_combined_counter
 827               (cm, cpu_index, lbi0, 1,
 828                vlib_buffer_length_in_chain (vm, p0));
 829           vlib_increment_combined_counter
 830               (cm, cpu_index, lbi1, 1,
 831                vlib_buffer_length_in_chain (vm, p1));
 832
 833           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 834                                            to_next, n_left_to_next,
 835                                            pi0, pi1, next0, next1);
 836        }
 837
 838       while (n_left_from > 0 && n_left_to_next > 0)
 839         {
 840           ip_lookup_next_t next0;
 841           const load_balance_t *lb0;
 842           vlib_buffer_t * p0;
 843           u32 pi0, lbi0, hc0;
 844           const ip6_header_t *ip0;
 845           const dpo_id_t *dpo0;
 846
 847           pi0 = from[0];
 848           to_next[0] = pi0;
 849           from += 1;
 850           to_next += 1;
 851           n_left_to_next -= 1;
 852           n_left_from -= 1;
 853
 854           p0 = vlib_get_buffer (vm, pi0);
 855
 856           ip0 = vlib_buffer_get_current (p0);
 857           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 858
 859           lb0 = load_balance_get(lbi0);
 860
 861           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 862           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 863           {
 864               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 865               {
 866                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 867               }
 868               else
 869               {
 870                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 871               }
 872           }
 873           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 874
 875           next0 = dpo0->dpoi_next_node;
 876           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 877
 878           /* Only process the HBH Option Header if explicitly configured to do so */
 879           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 880             {
 881               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 882                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 883             }
 884
 885           vlib_increment_combined_counter
 886               (cm, cpu_index, lbi0, 1,
 887                vlib_buffer_length_in_chain (vm, p0));
 888
 889           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 890                                            to_next, n_left_to_next,
 891                                            pi0, next0);
 892         }
 893
 894       vlib_put_next_frame (vm, node, next, n_left_to_next);
 895     }
 896
 897   return frame->n_vectors;
 898 }
 899
 900 VLIB_REGISTER_NODE (ip6_load_balance_node) = {
 901   .function = ip6_load_balance,
 902   .name = "ip6-load-balance",
 903   .vector_size = sizeof (u32),
 904   .sibling_of = "ip6-lookup",
 905
 906   .format_trace = format_ip6_lookup_trace,
 907 };
 908
 909 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance)
 910
 911 typedef struct {
 912   /* Adjacency taken. */
 913   u32 adj_index;
 914   u32 flow_hash;
 915   u32 fib_index;
 916
 917   /* Packet data, possibly *after* rewrite. */
 918   u8 packet_data[128 - 1*sizeof(u32)];
 919 } ip6_forward_next_trace_t;
 920
 921 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
 922 {
 923   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 924   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 925   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 926   uword indent = format_get_indent (s);
 927
 928   s = format(s, "%U%U",
 929              format_white_space, indent,
 930              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 931   return s;
 932 }
 933
 934 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
 935 {
 936   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 937   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 938   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 939   uword indent = format_get_indent (s);
 940
 941   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 942               t->fib_index, t->adj_index, t->flow_hash);
 943   s = format(s, "\n%U%U",
 944              format_white_space, indent,
 945              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 946   return s;
 947 }
 948
 949
 950 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
 951 {
 952   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 953   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 954   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 955   vnet_main_t * vnm = vnet_get_main();
 956   uword indent = format_get_indent (s);
 957
 958   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 959               t->fib_index, t->adj_index, format_ip_adjacency,
 960               t->adj_index, FORMAT_IP_ADJACENCY_NONE,
 961               t->flow_hash);
 962   s = format (s, "\n%U%U",
 963               format_white_space, indent,
 964               format_ip_adjacency_packet_data,
 965               vnm, t->adj_index,
 966               t->packet_data, sizeof (t->packet_data));
 967   return s;
 968 }
 969
 970 /* Common trace function for all ip6-forward next nodes. */
 971 void
 972 ip6_forward_next_trace (vlib_main_t * vm,
 973                         vlib_node_runtime_t * node,
 974                         vlib_frame_t * frame,
 975                         vlib_rx_or_tx_t which_adj_index)
 976 {
 977   u32 * from, n_left;
 978   ip6_main_t * im = &ip6_main;
 979
 980   n_left = frame->n_vectors;
 981   from = vlib_frame_vector_args (frame);
 982
 983   while (n_left >= 4)
 984     {
 985       u32 bi0, bi1;
 986       vlib_buffer_t * b0, * b1;
 987       ip6_forward_next_trace_t * t0, * t1;
 988
 989       /* Prefetch next iteration. */
 990       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 991       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 992
 993       bi0 = from[0];
 994       bi1 = from[1];
 995
 996       b0 = vlib_get_buffer (vm, bi0);
 997       b1 = vlib_get_buffer (vm, bi1);
 998
 999       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1000         {
1001           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1002           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1003           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1004           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1005               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1006               vec_elt (im->fib_index_by_sw_if_index,
1007                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1008
1009           clib_memcpy (t0->packet_data,
1010                   vlib_buffer_get_current (b0),
1011                   sizeof (t0->packet_data));
1012         }
1013       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1014         {
1015           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1016           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1017           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1018           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1019               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1020               vec_elt (im->fib_index_by_sw_if_index,
1021                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1022
1023           clib_memcpy (t1->packet_data,
1024                   vlib_buffer_get_current (b1),
1025                   sizeof (t1->packet_data));
1026         }
1027       from += 2;
1028       n_left -= 2;
1029     }
1030
1031   while (n_left >= 1)
1032     {
1033       u32 bi0;
1034       vlib_buffer_t * b0;
1035       ip6_forward_next_trace_t * t0;
1036
1037       bi0 = from[0];
1038
1039       b0 = vlib_get_buffer (vm, bi0);
1040
1041       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1042         {
1043           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1044           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1045           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1046           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1047               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1048               vec_elt (im->fib_index_by_sw_if_index,
1049                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1050
1051           clib_memcpy (t0->packet_data,
1052                   vlib_buffer_get_current (b0),
1053                   sizeof (t0->packet_data));
1054         }
1055       from += 1;
1056       n_left -= 1;
1057     }
1058 }
1059
1060 static uword
1061 ip6_drop_or_punt (vlib_main_t * vm,
1062                   vlib_node_runtime_t * node,
1063                   vlib_frame_t * frame,
1064                   ip6_error_t error_code)
1065 {
1066   u32 * buffers = vlib_frame_vector_args (frame);
1067   uword n_packets = frame->n_vectors;
1068
1069   vlib_error_drop_buffers (vm, node,
1070                            buffers,
1071                            /* stride */ 1,
1072                            n_packets,
1073                            /* next */ 0,
1074                            ip6_input_node.index,
1075                            error_code);
1076
1077   if (node->flags & VLIB_NODE_FLAG_TRACE)
1078     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1079
1080   return n_packets;
1081 }
1082
1083 static uword
1084 ip6_drop (vlib_main_t * vm,
1085           vlib_node_runtime_t * node,
1086           vlib_frame_t * frame)
1087 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1088
1089 static uword
1090 ip6_punt (vlib_main_t * vm,
1091           vlib_node_runtime_t * node,
1092           vlib_frame_t * frame)
1093 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1094
1095 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1096   .function = ip6_drop,
1097   .name = "ip6-drop",
1098   .vector_size = sizeof (u32),
1099
1100   .format_trace = format_ip6_forward_next_trace,
1101
1102   .n_next_nodes = 1,
1103   .next_nodes = {
1104     [0] = "error-drop",
1105   },
1106 };
1107
1108 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1109
1110 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1111   .function = ip6_punt,
1112   .name = "ip6-punt",
1113   .vector_size = sizeof (u32),
1114
1115   .format_trace = format_ip6_forward_next_trace,
1116
1117   .n_next_nodes = 1,
1118   .next_nodes = {
1119     [0] = "error-punt",
1120   },
1121 };
1122
1123 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1124
1125 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1126   .function = ip6_drop,
1127   .name = "ip6-multicast",
1128   .vector_size = sizeof (u32),
1129
1130   .format_trace = format_ip6_forward_next_trace,
1131
1132   .n_next_nodes = 1,
1133   .next_nodes = {
1134     [0] = "error-drop",
1135   },
1136 };
1137
1138 /* Compute TCP/UDP/ICMP6 checksum in software. */
1139 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1140 {
1141   ip_csum_t sum0;
1142   u16 sum16, payload_length_host_byte_order;
1143   u32 i, n_this_buffer, n_bytes_left;
1144   u32 headers_size = sizeof(ip0[0]);
1145   void * data_this_buffer;
1146
1147   ASSERT(bogus_lengthp);
1148   *bogus_lengthp = 0;
1149
1150   /* Initialize checksum with ip header. */
1151   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1152   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1153   data_this_buffer = (void *) (ip0 + 1);
1154
1155   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1156     {
1157       sum0 = ip_csum_with_carry (sum0,
1158                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1159       sum0 = ip_csum_with_carry (sum0,
1160                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1161     }
1162
1163   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1164   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1165     {
1166       u32  skip_bytes;
1167       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1168
1169       /* validate really icmp6 next */
1170       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1171
1172       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1173       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1174
1175       payload_length_host_byte_order  -= skip_bytes;
1176       headers_size += skip_bytes;
1177    }
1178
1179   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1180   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1181     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1182   while (1)
1183     {
1184       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1185       n_bytes_left -= n_this_buffer;
1186       if (n_bytes_left == 0)
1187         break;
1188
1189       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1190         {
1191           *bogus_lengthp = 1;
1192           return 0xfefe;
1193         }
1194       p0 = vlib_get_buffer (vm, p0->next_buffer);
1195       data_this_buffer = vlib_buffer_get_current (p0);
1196       n_this_buffer = p0->current_length;
1197     }
1198
1199   sum16 = ~ ip_csum_fold (sum0);
1200
1201   return sum16;
1202 }
1203
1204 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1205 {
1206   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1207   udp_header_t * udp0;
1208   u16 sum16;
1209   int bogus_length;
1210
1211   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1212   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1213           || ip0->protocol == IP_PROTOCOL_ICMP6
1214           || ip0->protocol == IP_PROTOCOL_UDP
1215           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1216
1217   udp0 = (void *) (ip0 + 1);
1218   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1219     {
1220       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1221                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1222       return p0->flags;
1223     }
1224
1225   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1226
1227   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1228                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1229
1230   return p0->flags;
1231 }
1232
1233 static uword
1234 ip6_local (vlib_main_t * vm,
1235            vlib_node_runtime_t * node,
1236            vlib_frame_t * frame)
1237 {
1238   ip6_main_t * im = &ip6_main;
1239   ip_lookup_main_t * lm = &im->lookup_main;
1240   ip_local_next_t next_index;
1241   u32 * from, * to_next, n_left_from, n_left_to_next;
1242   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1243
1244   from = vlib_frame_vector_args (frame);
1245   n_left_from = frame->n_vectors;
1246   next_index = node->cached_next_index;
1247
1248   if (node->flags & VLIB_NODE_FLAG_TRACE)
1249     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1250
1251   while (n_left_from > 0)
1252     {
1253       vlib_get_next_frame (vm, node, next_index,
1254                            to_next, n_left_to_next);
1255
1256       while (n_left_from >= 4 && n_left_to_next >= 2)
1257         {
1258           vlib_buffer_t * p0, * p1;
1259           ip6_header_t * ip0, * ip1;
1260           udp_header_t * udp0, * udp1;
1261           u32 pi0, ip_len0, udp_len0, flags0, next0;
1262           u32 pi1, ip_len1, udp_len1, flags1, next1;
1263           i32 len_diff0, len_diff1;
1264           u8 error0, type0, good_l4_checksum0;
1265           u8 error1, type1, good_l4_checksum1;
1266
1267           pi0 = to_next[0] = from[0];
1268           pi1 = to_next[1] = from[1];
1269           from += 2;
1270           n_left_from -= 2;
1271           to_next += 2;
1272           n_left_to_next -= 2;
1273
1274           p0 = vlib_get_buffer (vm, pi0);
1275           p1 = vlib_get_buffer (vm, pi1);
1276
1277           ip0 = vlib_buffer_get_current (p0);
1278           ip1 = vlib_buffer_get_current (p1);
1279
1280           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1281           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1282
1283           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1284           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1285
1286           flags0 = p0->flags;
1287           flags1 = p1->flags;
1288
1289           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1290           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1291
1292           udp0 = ip6_next_header (ip0);
1293           udp1 = ip6_next_header (ip1);
1294
1295           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1296           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1297           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1298
1299           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1300           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1301
1302           /* Verify UDP length. */
1303           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1304           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1305           udp_len0 = clib_net_to_host_u16 (udp0->length);
1306           udp_len1 = clib_net_to_host_u16 (udp1->length);
1307
1308           len_diff0 = ip_len0 - udp_len0;
1309           len_diff1 = ip_len1 - udp_len1;
1310
1311           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1312           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1313
1314           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1315                              && ! good_l4_checksum0
1316                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1317             {
1318               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1319               good_l4_checksum0 =
1320                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1321             }
1322           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1323                              && ! good_l4_checksum1
1324                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1325             {
1326               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1327               good_l4_checksum1 =
1328                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1329             }
1330
1331           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1332
1333           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1334           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1335
1336           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1337           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1338           error0 = (! good_l4_checksum0
1339                     ? IP6_ERROR_UDP_CHECKSUM + type0
1340                     : error0);
1341           error1 = (! good_l4_checksum1
1342                     ? IP6_ERROR_UDP_CHECKSUM + type1
1343                     : error1);
1344
1345           /* Drop packets from unroutable hosts. */
1346           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1347           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1348               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1349               !ip6_address_is_link_local_unicast(&ip0->src_address))
1350             {
1351               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1352               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1353                         ? IP6_ERROR_SRC_LOOKUP_MISS
1354                         : error0);
1355             }
1356           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1357               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1358               !ip6_address_is_link_local_unicast(&ip1->src_address))
1359             {
1360               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1361               error1 = (ADJ_INDEX_INVALID == src_adj_index1
1362                         ? IP6_ERROR_SRC_LOOKUP_MISS
1363                         : error1);
1364             }
1365
1366           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1367           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1368
1369           p0->error = error_node->errors[error0];
1370           p1->error = error_node->errors[error1];
1371
1372           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1373                                            to_next, n_left_to_next,
1374                                            pi0, pi1, next0, next1);
1375         }
1376
1377       while (n_left_from > 0 && n_left_to_next > 0)
1378         {
1379           vlib_buffer_t * p0;
1380           ip6_header_t * ip0;
1381           udp_header_t * udp0;
1382           u32 pi0, ip_len0, udp_len0, flags0, next0;
1383           i32 len_diff0;
1384           u8 error0, type0, good_l4_checksum0;
1385
1386           pi0 = to_next[0] = from[0];
1387           from += 1;
1388           n_left_from -= 1;
1389           to_next += 1;
1390           n_left_to_next -= 1;
1391
1392           p0 = vlib_get_buffer (vm, pi0);
1393
1394           ip0 = vlib_buffer_get_current (p0);
1395
1396           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1397           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1398
1399           flags0 = p0->flags;
1400
1401           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1402
1403           udp0 = ip6_next_header (ip0);
1404
1405           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1406           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1407
1408           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1409
1410           /* Verify UDP length. */
1411           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1412           udp_len0 = clib_net_to_host_u16 (udp0->length);
1413
1414           len_diff0 = ip_len0 - udp_len0;
1415
1416           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1417
1418           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1419                              && ! good_l4_checksum0
1420                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1421             {
1422               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1423               good_l4_checksum0 =
1424                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1425             }
1426
1427           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1428
1429           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1430
1431           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1432           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1433           error0 = (! good_l4_checksum0
1434                     ? IP6_ERROR_UDP_CHECKSUM + type0
1435                     : error0);
1436
1437           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1438           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1439               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1440               !ip6_address_is_link_local_unicast(&ip0->src_address))
1441             {
1442               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1443               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1444                         ? IP6_ERROR_SRC_LOOKUP_MISS
1445                         : error0);
1446             }
1447
1448           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1449
1450           p0->error = error_node->errors[error0];
1451
1452           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1453                                            to_next, n_left_to_next,
1454                                            pi0, next0);
1455         }
1456
1457       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1458     }
1459
1460   return frame->n_vectors;
1461 }
1462
1463 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1464   .function = ip6_local,
1465   .name = "ip6-local",
1466   .vector_size = sizeof (u32),
1467
1468   .format_trace = format_ip6_forward_next_trace,
1469
1470   .n_next_nodes = IP_LOCAL_N_NEXT,
1471   .next_nodes = {
1472     [IP_LOCAL_NEXT_DROP] = "error-drop",
1473     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1474     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1475     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1476   },
1477 };
1478
1479 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
1480
1481 void ip6_register_protocol (u32 protocol, u32 node_index)
1482 {
1483   vlib_main_t * vm = vlib_get_main();
1484   ip6_main_t * im = &ip6_main;
1485   ip_lookup_main_t * lm = &im->lookup_main;
1486
1487   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1488   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1489 }
1490
1491 typedef enum {
1492   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1493   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1494   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1495 } ip6_discover_neighbor_next_t;
1496
1497 typedef enum {
1498   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1499   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1500   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
1501 } ip6_discover_neighbor_error_t;
1502
1503 static uword
1504 ip6_discover_neighbor_inline (vlib_main_t * vm,
1505                               vlib_node_runtime_t * node,
1506                               vlib_frame_t * frame,
1507                               int is_glean)
1508 {
1509   vnet_main_t * vnm = vnet_get_main();
1510   ip6_main_t * im = &ip6_main;
1511   ip_lookup_main_t * lm = &im->lookup_main;
1512   u32 * from, * to_next_drop;
1513   uword n_left_from, n_left_to_next_drop;
1514   static f64 time_last_seed_change = -1e100;
1515   static u32 hash_seeds[3];
1516   static uword hash_bitmap[256 / BITS (uword)];
1517   f64 time_now;
1518   int bogus_length;
1519
1520   if (node->flags & VLIB_NODE_FLAG_TRACE)
1521     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1522
1523   time_now = vlib_time_now (vm);
1524   if (time_now - time_last_seed_change > 1e-3)
1525     {
1526       uword i;
1527       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1528                                              sizeof (hash_seeds));
1529       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1530         hash_seeds[i] = r[i];
1531
1532       /* Mark all hash keys as been not-seen before. */
1533       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1534         hash_bitmap[i] = 0;
1535
1536       time_last_seed_change = time_now;
1537     }
1538
1539   from = vlib_frame_vector_args (frame);
1540   n_left_from = frame->n_vectors;
1541
1542   while (n_left_from > 0)
1543     {
1544       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1545                            to_next_drop, n_left_to_next_drop);
1546
1547       while (n_left_from > 0 && n_left_to_next_drop > 0)
1548         {
1549           vlib_buffer_t * p0;
1550           ip6_header_t * ip0;
1551           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1552           uword bm0;
1553           ip_adjacency_t * adj0;
1554           vnet_hw_interface_t * hw_if0;
1555           u32 next0;
1556
1557           pi0 = from[0];
1558
1559           p0 = vlib_get_buffer (vm, pi0);
1560
1561           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1562
1563           ip0 = vlib_buffer_get_current (p0);
1564
1565           adj0 = ip_get_adjacency (lm, adj_index0);
1566
1567           if (!is_glean)
1568             {
1569               ip0->dst_address.as_u64[0] = adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
1570               ip0->dst_address.as_u64[1] = adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
1571             }
1572
1573           a0 = hash_seeds[0];
1574           b0 = hash_seeds[1];
1575           c0 = hash_seeds[2];
1576
1577           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1578           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1579
1580           a0 ^= sw_if_index0;
1581           b0 ^= ip0->dst_address.as_u32[0];
1582           c0 ^= ip0->dst_address.as_u32[1];
1583
1584           hash_v3_mix32 (a0, b0, c0);
1585
1586           b0 ^= ip0->dst_address.as_u32[2];
1587           c0 ^= ip0->dst_address.as_u32[3];
1588
1589           hash_v3_finalize32 (a0, b0, c0);
1590
1591           c0 &= BITS (hash_bitmap) - 1;
1592           c0 = c0 / BITS (uword);
1593           m0 = (uword) 1 << (c0 % BITS (uword));
1594
1595           bm0 = hash_bitmap[c0];
1596           drop0 = (bm0 & m0) != 0;
1597
1598           /* Mark it as seen. */
1599           hash_bitmap[c0] = bm0 | m0;
1600
1601           from += 1;
1602           n_left_from -= 1;
1603           to_next_drop[0] = pi0;
1604           to_next_drop += 1;
1605           n_left_to_next_drop -= 1;
1606
1607           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1608
1609           /* If the interface is link-down, drop the pkt */
1610           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1611             drop0 = 1;
1612
1613           p0->error =
1614             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1615                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1616           if (drop0)
1617             continue;
1618
1619           /*
1620            * the adj has been updated to a rewrite but the node the DPO that got
1621            * us here hasn't - yet. no big deal. we'll drop while we wait.
1622            */
1623           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1624             continue;
1625
1626           {
1627             u32 bi0 = 0;
1628             icmp6_neighbor_solicitation_header_t * h0;
1629             vlib_buffer_t * b0;
1630
1631             h0 = vlib_packet_template_get_packet
1632               (vm, &im->discover_neighbor_packet_template, &bi0);
1633
1634             /*
1635              * Build ethernet header.
1636              * Choose source address based on destination lookup
1637              * adjacency.
1638              */
1639             if (ip6_src_address_for_packet (lm,
1640                                             sw_if_index0,
1641                                             &h0->ip.src_address))
1642               {
1643                 /* There is no address on the interface */
1644                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
1645                 vlib_buffer_free(vm, &bi0, 1);
1646                 continue;
1647               }
1648
1649             /*
1650              * Destination address is a solicited node multicast address.
1651              * We need to fill in
1652              * the low 24 bits with low 24 bits of target's address.
1653              */
1654             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1655             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1656             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1657
1658             h0->neighbor.target_address = ip0->dst_address;
1659
1660             clib_memcpy (h0->link_layer_option.ethernet_address,
1661                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
1662
1663             /* $$$$ appears we need this; why is the checksum non-zero? */
1664             h0->neighbor.icmp.checksum = 0;
1665             h0->neighbor.icmp.checksum =
1666               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1667                                                  &bogus_length);
1668
1669             ASSERT (bogus_length == 0);
1670
1671             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1672             b0 = vlib_get_buffer (vm, bi0);
1673             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1674               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1675
1676             /* Add rewrite/encap string. */
1677             vnet_rewrite_one_header (adj0[0], h0,
1678                                      sizeof (ethernet_header_t));
1679             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1680
1681             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1682
1683             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1684           }
1685         }
1686
1687       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1688                            n_left_to_next_drop);
1689     }
1690
1691   return frame->n_vectors;
1692 }
1693
1694 static uword
1695 ip6_discover_neighbor (vlib_main_t * vm,
1696                        vlib_node_runtime_t * node,
1697                        vlib_frame_t * frame)
1698 {
1699     return (ip6_discover_neighbor_inline(vm, node, frame, 0));
1700 }
1701
1702 static uword
1703 ip6_glean (vlib_main_t * vm,
1704            vlib_node_runtime_t * node,
1705            vlib_frame_t * frame)
1706 {
1707     return (ip6_discover_neighbor_inline(vm, node, frame, 1));
1708 }
1709
1710 static char * ip6_discover_neighbor_error_strings[] = {
1711   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1712   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
1713   = "neighbor solicitations sent",
1714   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
1715     = "no source address for ND solicitation",
1716 };
1717
1718 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
1719   .function = ip6_discover_neighbor,
1720   .name = "ip6-discover-neighbor",
1721   .vector_size = sizeof (u32),
1722
1723   .format_trace = format_ip6_forward_next_trace,
1724
1725   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1726   .error_strings = ip6_discover_neighbor_error_strings,
1727
1728   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1729   .next_nodes = {
1730     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1731     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1732   },
1733 };
1734
1735 VLIB_REGISTER_NODE (ip6_glean_node) = {
1736   .function = ip6_glean,
1737   .name = "ip6-glean",
1738   .vector_size = sizeof (u32),
1739
1740   .format_trace = format_ip6_forward_next_trace,
1741
1742   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1743   .error_strings = ip6_discover_neighbor_error_strings,
1744
1745   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1746   .next_nodes = {
1747     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1748     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1749   },
1750 };
1751
1752 clib_error_t *
1753 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
1754 {
1755   vnet_main_t * vnm = vnet_get_main();
1756   ip6_main_t * im = &ip6_main;
1757   icmp6_neighbor_solicitation_header_t * h;
1758   ip6_address_t * src;
1759   ip_interface_address_t * ia;
1760   ip_adjacency_t * adj;
1761   vnet_hw_interface_t * hi;
1762   vnet_sw_interface_t * si;
1763   vlib_buffer_t * b;
1764   u32 bi = 0;
1765   int bogus_length;
1766
1767   si = vnet_get_sw_interface (vnm, sw_if_index);
1768
1769   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1770     {
1771       return clib_error_return (0, "%U: interface %U down",
1772                                 format_ip6_address, dst,
1773                                 format_vnet_sw_if_index_name, vnm,
1774                                 sw_if_index);
1775     }
1776
1777   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1778   if (! src)
1779     {
1780       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1781       return clib_error_return
1782         (0, "no matching interface address for destination %U (interface %U)",
1783          format_ip6_address, dst,
1784          format_vnet_sw_if_index_name, vnm, sw_if_index);
1785     }
1786
1787   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
1788
1789   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1790
1791   /* Destination address is a solicited node multicast address.  We need to fill in
1792      the low 24 bits with low 24 bits of target's address. */
1793   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1794   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1795   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1796
1797   h->ip.src_address = src[0];
1798   h->neighbor.target_address = dst[0];
1799
1800   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
1801
1802   h->neighbor.icmp.checksum =
1803     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1804   ASSERT(bogus_length == 0);
1805
1806   b = vlib_get_buffer (vm, bi);
1807   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1808
1809   /* Add encapsulation string for software interface (e.g. ethernet header). */
1810   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
1811   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1812   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1813
1814   {
1815     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
1816     u32 * to_next = vlib_frame_vector_args (f);
1817     to_next[0] = bi;
1818     f->n_vectors = 1;
1819     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1820   }
1821
1822   return /* no error */ 0;
1823 }
1824
1825 typedef enum {
1826   IP6_REWRITE_NEXT_DROP,
1827   IP6_REWRITE_NEXT_ICMP_ERROR,
1828 } ip6_rewrite_next_t;
1829
1830 always_inline uword
1831 ip6_rewrite_inline (vlib_main_t * vm,
1832                     vlib_node_runtime_t * node,
1833                     vlib_frame_t * frame,
1834                     int is_midchain)
1835 {
1836   ip_lookup_main_t * lm = &ip6_main.lookup_main;
1837   u32 * from = vlib_frame_vector_args (frame);
1838   u32 n_left_from, n_left_to_next, * to_next, next_index;
1839   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1840
1841   n_left_from = frame->n_vectors;
1842   next_index = node->cached_next_index;
1843   u32 cpu_index = os_get_cpu_number();
1844
1845   while (n_left_from > 0)
1846     {
1847       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1848
1849       while (n_left_from >= 4 && n_left_to_next >= 2)
1850         {
1851           ip_adjacency_t * adj0, * adj1;
1852           vlib_buffer_t * p0, * p1;
1853           ip6_header_t * ip0, * ip1;
1854           u32 pi0, rw_len0, next0, error0, adj_index0;
1855           u32 pi1, rw_len1, next1, error1, adj_index1;
1856           u32 tx_sw_if_index0, tx_sw_if_index1;
1857
1858           /* Prefetch next iteration. */
1859           {
1860             vlib_buffer_t * p2, * p3;
1861
1862             p2 = vlib_get_buffer (vm, from[2]);
1863             p3 = vlib_get_buffer (vm, from[3]);
1864
1865             vlib_prefetch_buffer_header (p2, LOAD);
1866             vlib_prefetch_buffer_header (p3, LOAD);
1867
1868             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1869             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1870
1871             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1872             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1873           }
1874
1875           pi0 = to_next[0] = from[0];
1876           pi1 = to_next[1] = from[1];
1877
1878           from += 2;
1879           n_left_from -= 2;
1880           to_next += 2;
1881           n_left_to_next -= 2;
1882
1883           p0 = vlib_get_buffer (vm, pi0);
1884           p1 = vlib_get_buffer (vm, pi1);
1885
1886           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1887           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1888
1889           /* We should never rewrite a pkt using the MISS adjacency */
1890           ASSERT(adj_index0 && adj_index1);
1891
1892           ip0 = vlib_buffer_get_current (p0);
1893           ip1 = vlib_buffer_get_current (p1);
1894
1895           error0 = error1 = IP6_ERROR_NONE;
1896           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1897
1898           if (PREDICT_TRUE(!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
1899             {
1900                 i32 hop_limit0 = ip0->hop_limit;
1901
1902               /* Input node should have reject packets with hop limit 0. */
1903               ASSERT (ip0->hop_limit > 0);
1904
1905               hop_limit0 -= 1;
1906
1907               ip0->hop_limit = hop_limit0;
1908
1909               /*
1910                * If the hop count drops below 1 when forwarding, generate
1911                * an ICMP response.
1912                */
1913               if (PREDICT_FALSE(hop_limit0 <= 0))
1914                 {
1915                   error0 = IP6_ERROR_TIME_EXPIRED;
1916                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1917                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
1918                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
1919                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1920                 }
1921             }
1922           else
1923             {
1924               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
1925             }
1926           if (PREDICT_TRUE(!(p1->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
1927           {
1928               i32 hop_limit1 = ip1->hop_limit;
1929
1930               /* Input node should have reject packets with hop limit 0. */
1931               ASSERT (ip1->hop_limit > 0);
1932
1933               hop_limit1 -= 1;
1934
1935               ip1->hop_limit = hop_limit1;
1936
1937               /*
1938                * If the hop count drops below 1 when forwarding, generate
1939                * an ICMP response.
1940                */
1941               if (PREDICT_FALSE(hop_limit1 <= 0))
1942                 {
1943                   error1 = IP6_ERROR_TIME_EXPIRED;
1944                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1945                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
1946                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
1947                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1948                 }
1949           }
1950           else
1951             {
1952               p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
1953             }
1954           adj0 = ip_get_adjacency (lm, adj_index0);
1955           adj1 = ip_get_adjacency (lm, adj_index1);
1956
1957           rw_len0 = adj0[0].rewrite_header.data_bytes;
1958           rw_len1 = adj1[0].rewrite_header.data_bytes;
1959           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
1960           vnet_buffer(p1)->ip.save_rewrite_length = rw_len1;
1961
1962           vlib_increment_combined_counter (&adjacency_counters,
1963                                            cpu_index,
1964                                            adj_index0,
1965                                            /* packet increment */ 0,
1966                                            /* byte increment */ rw_len0);
1967           vlib_increment_combined_counter (&adjacency_counters,
1968                                            cpu_index,
1969                                            adj_index1,
1970                                            /* packet increment */ 0,
1971                                            /* byte increment */ rw_len1);
1972
1973           /* Check MTU of outgoing interface. */
1974           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
1975                     ? IP6_ERROR_MTU_EXCEEDED
1976                     : error0);
1977           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
1978                     ? IP6_ERROR_MTU_EXCEEDED
1979                     : error1);
1980
1981           /* Don't adjust the buffer for hop count issue; icmp-error node
1982            * wants to see the IP headerr */
1983           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
1984             {
1985               p0->current_data -= rw_len0;
1986               p0->current_length += rw_len0;
1987
1988               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1989               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
1990                   tx_sw_if_index0;
1991               next0 = adj0[0].rewrite_header.next_index;
1992
1993               vnet_feature_arc_start(lm->output_feature_arc_index,
1994                                      tx_sw_if_index0, &next0, p0);
1995             }
1996           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
1997             {
1998               p1->current_data -= rw_len1;
1999               p1->current_length += rw_len1;
2000
2001               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2002               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2003                   tx_sw_if_index1;
2004               next1 = adj1[0].rewrite_header.next_index;
2005
2006               vnet_feature_arc_start(lm->output_feature_arc_index,
2007                                      tx_sw_if_index1, &next1, p1);
2008             }
2009
2010           /* Guess we are only writing on simple Ethernet header. */
2011           vnet_rewrite_two_headers (adj0[0], adj1[0],
2012                                     ip0, ip1,
2013                                     sizeof (ethernet_header_t));
2014
2015           if (is_midchain)
2016           {
2017               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2018               adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
2019           }
2020
2021           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2022                                            to_next, n_left_to_next,
2023                                            pi0, pi1, next0, next1);
2024         }
2025
2026       while (n_left_from > 0 && n_left_to_next > 0)
2027         {
2028           ip_adjacency_t * adj0;
2029           vlib_buffer_t * p0;
2030           ip6_header_t * ip0;
2031           u32 pi0, rw_len0;
2032           u32 adj_index0, next0, error0;
2033           u32 tx_sw_if_index0;
2034
2035           pi0 = to_next[0] = from[0];
2036
2037           p0 = vlib_get_buffer (vm, pi0);
2038
2039           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2040
2041           /* We should never rewrite a pkt using the MISS adjacency */
2042           ASSERT(adj_index0);
2043
2044           adj0 = ip_get_adjacency (lm, adj_index0);
2045
2046           ip0 = vlib_buffer_get_current (p0);
2047
2048           error0 = IP6_ERROR_NONE;
2049           next0 = IP6_REWRITE_NEXT_DROP;
2050
2051           /* Check hop limit */
2052           if (PREDICT_TRUE(!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2053             {
2054               i32 hop_limit0 = ip0->hop_limit;
2055
2056               ASSERT (ip0->hop_limit > 0);
2057
2058               hop_limit0 -= 1;
2059
2060               ip0->hop_limit = hop_limit0;
2061
2062               if (PREDICT_FALSE(hop_limit0 <= 0))
2063                 {
2064                   /*
2065                    * If the hop count drops below 1 when forwarding, generate
2066                    * an ICMP response.
2067                    */
2068                   error0 = IP6_ERROR_TIME_EXPIRED;
2069                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2070                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2071                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2072                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2073                 }
2074             }
2075           else
2076             {
2077               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2078             }
2079
2080           /* Guess we are only writing on simple Ethernet header. */
2081           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2082
2083           /* Update packet buffer attributes/set output interface. */
2084           rw_len0 = adj0[0].rewrite_header.data_bytes;
2085           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2086
2087           vlib_increment_combined_counter (&adjacency_counters,
2088                                            cpu_index,
2089                                            adj_index0,
2090                                            /* packet increment */ 0,
2091                                            /* byte increment */ rw_len0);
2092
2093           /* Check MTU of outgoing interface. */
2094           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2095                     ? IP6_ERROR_MTU_EXCEEDED
2096                     : error0);
2097
2098           /* Don't adjust the buffer for hop count issue; icmp-error node
2099            * wants to see the IP headerr */
2100           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2101             {
2102               p0->current_data -= rw_len0;
2103               p0->current_length += rw_len0;
2104
2105               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2106
2107               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2108               next0 = adj0[0].rewrite_header.next_index;
2109
2110               vnet_feature_arc_start(lm->output_feature_arc_index,
2111                                      tx_sw_if_index0, &next0, p0);
2112             }
2113
2114           if (is_midchain)
2115           {
2116               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2117           }
2118
2119           p0->error = error_node->errors[error0];
2120
2121           from += 1;
2122           n_left_from -= 1;
2123           to_next += 1;
2124           n_left_to_next -= 1;
2125
2126           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2127                                            to_next, n_left_to_next,
2128                                            pi0, next0);
2129         }
2130
2131       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2132     }
2133
2134   /* Need to do trace after rewrites to pick up new packet data. */
2135   if (node->flags & VLIB_NODE_FLAG_TRACE)
2136     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2137
2138   return frame->n_vectors;
2139 }
2140
2141 static uword
2142 ip6_rewrite (vlib_main_t * vm,
2143              vlib_node_runtime_t * node,
2144              vlib_frame_t * frame)
2145 {
2146   return ip6_rewrite_inline (vm, node, frame,
2147                              /* midchain */ 0);
2148 }
2149
2150 static uword
2151 ip6_midchain (vlib_main_t * vm,
2152               vlib_node_runtime_t * node,
2153               vlib_frame_t * frame)
2154 {
2155   return ip6_rewrite_inline (vm, node, frame,
2156                              /* midchain */ 1);
2157 }
2158
2159 VLIB_REGISTER_NODE (ip6_midchain_node) = {
2160   .function = ip6_midchain,
2161   .name = "ip6-midchain",
2162   .vector_size = sizeof (u32),
2163
2164   .format_trace = format_ip6_forward_next_trace,
2165
2166   .sibling_of = "ip6-rewrite",
2167 };
2168
2169 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain)
2170
2171 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2172   .function = ip6_rewrite,
2173   .name = "ip6-rewrite",
2174   .vector_size = sizeof (u32),
2175
2176   .format_trace = format_ip6_rewrite_trace,
2177
2178   .n_next_nodes = 2,
2179   .next_nodes = {
2180     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2181     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2182   },
2183 };
2184
2185 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite);
2186
2187 /*
2188  * Hop-by-Hop handling
2189  */
2190
2191 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2192
2193 #define foreach_ip6_hop_by_hop_error \
2194 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2195 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2196 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2197
2198 typedef enum {
2199 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2200   foreach_ip6_hop_by_hop_error
2201 #undef _
2202   IP6_HOP_BY_HOP_N_ERROR,
2203 } ip6_hop_by_hop_error_t;
2204
2205 /*
2206  * Primary h-b-h handler trace support
2207  * We work pretty hard on the problem for obvious reasons
2208  */
2209 typedef struct {
2210   u32 next_index;
2211   u32 trace_len;
2212   u8 option_data[256];
2213 } ip6_hop_by_hop_trace_t;
2214
2215 vlib_node_registration_t ip6_hop_by_hop_node;
2216
2217 static char * ip6_hop_by_hop_error_strings[] = {
2218 #define _(sym,string) string,
2219   foreach_ip6_hop_by_hop_error
2220 #undef _
2221 };
2222
2223 static u8 *
2224 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2225 {
2226   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2227   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2228   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2229   ip6_hop_by_hop_header_t *hbh0;
2230   ip6_hop_by_hop_option_t *opt0, *limit0;
2231   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2232
2233   u8 type0;
2234
2235   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2236
2237   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2238               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2239
2240   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2241   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2242
2243   while (opt0 < limit0) {
2244     type0 = opt0->type;
2245     switch (type0) {
2246     case 0: /* Pad, just stop */
2247       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2248       break;
2249
2250     default:
2251       if (hm->trace[type0]) {
2252         s = (*hm->trace[type0])(s, opt0);
2253       } else {
2254         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2255       }
2256       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2257       break;
2258     }
2259   }
2260   return s;
2261 }
2262
2263 always_inline u8 ip6_scan_hbh_options (
2264                                        vlib_buffer_t * b0,
2265                                        ip6_header_t *ip0,
2266                                        ip6_hop_by_hop_header_t *hbh0,
2267                                        ip6_hop_by_hop_option_t *opt0,
2268                                        ip6_hop_by_hop_option_t *limit0,
2269                                        u32 *next0)
2270 {
2271   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2272   u8 type0;
2273   u8 error0 = 0;
2274
2275   while (opt0 < limit0)
2276     {
2277       type0 = opt0->type;
2278       switch (type0)
2279         {
2280         case 0: /* Pad1 */
2281           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2282           continue;
2283         case 1: /* PadN */
2284           break;
2285         default:
2286           if (hm->options[type0])
2287             {
2288               if ((*hm->options[type0])(b0, ip0, opt0) < 0)
2289                 {
2290                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2291                   return(error0);
2292                 }
2293             }
2294           else
2295             {
2296               /* Unrecognized mandatory option, check the two high order bits */
2297               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2298                 {
2299                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2300                   break;
2301                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2302                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2303                   *next0 = IP_LOOKUP_NEXT_DROP;
2304                   break;
2305                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2306                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2307                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2308                   icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2309                                               ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2310                   break;
2311                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2312                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2313                   if (!ip6_address_is_multicast(&ip0->dst_address))
2314                     {
2315                       *next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2316                       icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2317                                                   ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2318                     }
2319                   else
2320                     {
2321                       *next0 =  IP_LOOKUP_NEXT_DROP;
2322                     }
2323                   break;
2324                 }
2325               return(error0);
2326             }
2327         }
2328       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2329     }
2330   return(error0);
2331 }
2332
2333 /*
2334  * Process the Hop-by-Hop Options header
2335  */
2336 static uword
2337 ip6_hop_by_hop (vlib_main_t * vm,
2338                 vlib_node_runtime_t * node,
2339                 vlib_frame_t * frame)
2340 {
2341   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2342   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2343   u32 n_left_from, *from, *to_next;
2344   ip_lookup_next_t next_index;
2345   ip6_main_t * im = &ip6_main;
2346   ip_lookup_main_t *lm = &im->lookup_main;
2347
2348   from = vlib_frame_vector_args (frame);
2349   n_left_from = frame->n_vectors;
2350   next_index = node->cached_next_index;
2351
2352   while (n_left_from > 0) {
2353     u32 n_left_to_next;
2354
2355     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2356
2357     while (n_left_from >= 4 && n_left_to_next >= 2) {
2358       u32 bi0, bi1;
2359       vlib_buffer_t * b0, *b1;
2360       u32 next0, next1;
2361       ip6_header_t * ip0, *ip1;
2362       ip6_hop_by_hop_header_t *hbh0, *hbh1;
2363       ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2364       u8 error0 = 0, error1 = 0;
2365
2366       /* Prefetch next iteration. */
2367       {
2368         vlib_buffer_t * p2, * p3;
2369
2370         p2 = vlib_get_buffer (vm, from[2]);
2371         p3 = vlib_get_buffer (vm, from[3]);
2372
2373         vlib_prefetch_buffer_header (p2, LOAD);
2374         vlib_prefetch_buffer_header (p3, LOAD);
2375
2376         CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2377         CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2378       }
2379
2380       /* Speculatively enqueue b0, b1 to the current next frame */
2381       to_next[0] = bi0 = from[0];
2382       to_next[1] = bi1 = from[1];
2383       from += 2;
2384       to_next += 2;
2385       n_left_from -= 2;
2386       n_left_to_next -= 2;
2387
2388       b0 = vlib_get_buffer (vm, bi0);
2389       b1 = vlib_get_buffer (vm, bi1);
2390
2391       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2392       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2393       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2394       u32 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
2395       ip_adjacency_t *adj1 = ip_get_adjacency(lm, adj_index1);
2396
2397       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2398       next0 = adj0->lookup_next_index;
2399       next1 = adj1->lookup_next_index;
2400
2401       ip0 = vlib_buffer_get_current (b0);
2402       ip1 = vlib_buffer_get_current (b1);
2403       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2404       hbh1 = (ip6_hop_by_hop_header_t *)(ip1+1);
2405       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2406       opt1 = (ip6_hop_by_hop_option_t *)(hbh1+1);
2407       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2408       limit1 = (ip6_hop_by_hop_option_t *)((u8 *)hbh1 + ((hbh1->length + 1) << 3));
2409
2410       /*
2411        * Basic validity checks
2412        */
2413       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2414         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2415         next0 = IP_LOOKUP_NEXT_DROP;
2416         goto outdual;
2417       }
2418       /* Scan the set of h-b-h options, process ones that we understand */
2419       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2420
2421       if ((hbh1->length + 1) << 3 > clib_net_to_host_u16(ip1->payload_length)) {
2422         error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2423         next1 = IP_LOOKUP_NEXT_DROP;
2424         goto outdual;
2425       }
2426       /* Scan the set of h-b-h options, process ones that we understand */
2427       error1 = ip6_scan_hbh_options(b1,ip1,hbh1,opt1,limit1, &next1);
2428
2429     outdual:
2430       /* Has the classifier flagged this buffer for special treatment? */
2431       if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2432         next0 = hm->next_override;
2433
2434       /* Has the classifier flagged this buffer for special treatment? */
2435       if (PREDICT_FALSE((error1 == 0) && (vnet_buffer(b1)->l2_classify.opaque_index & OI_DECAP)))
2436         next1 = hm->next_override;
2437
2438       if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
2439         {
2440           if (b0->flags & VLIB_BUFFER_IS_TRACED) {
2441             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2442             u32 trace_len = (hbh0->length + 1) << 3;
2443             t->next_index = next0;
2444             /* Capture the h-b-h option verbatim */
2445             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2446             t->trace_len = trace_len;
2447             clib_memcpy(t->option_data, hbh0, trace_len);
2448           }
2449           if (b1->flags & VLIB_BUFFER_IS_TRACED) {
2450             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b1, sizeof (*t));
2451             u32 trace_len = (hbh1->length + 1) << 3;
2452             t->next_index = next1;
2453             /* Capture the h-b-h option verbatim */
2454             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2455             t->trace_len = trace_len;
2456             clib_memcpy(t->option_data, hbh1, trace_len);
2457           }
2458
2459         }
2460
2461       b0->error = error_node->errors[error0];
2462       b1->error = error_node->errors[error1];
2463
2464       /* verify speculative enqueue, maybe switch current next frame */
2465       vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0,
2466                                        bi1,next0, next1);
2467     }
2468
2469     while (n_left_from > 0 && n_left_to_next > 0) {
2470       u32 bi0;
2471       vlib_buffer_t * b0;
2472       u32 next0;
2473       ip6_header_t * ip0;
2474       ip6_hop_by_hop_header_t *hbh0;
2475       ip6_hop_by_hop_option_t *opt0, *limit0;
2476       u8 error0 = 0;
2477
2478       /* Speculatively enqueue b0 to the current next frame */
2479       bi0 = from[0];
2480       to_next[0] = bi0;
2481       from += 1;
2482       to_next += 1;
2483       n_left_from -= 1;
2484       n_left_to_next -= 1;
2485
2486       b0 = vlib_get_buffer (vm, bi0);
2487       /*
2488        * Default use the next_index from the adjacency.
2489        * A HBH option rarely redirects to a different node
2490        */
2491       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2492       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2493       next0 = adj0->lookup_next_index;
2494
2495       ip0 = vlib_buffer_get_current (b0);
2496       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2497       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2498       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2499
2500       /*
2501        * Basic validity checks
2502        */
2503       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2504         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2505         next0 = IP_LOOKUP_NEXT_DROP;
2506         goto out0;
2507       }
2508
2509       /* Scan the set of h-b-h options, process ones that we understand */
2510       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2511
2512     out0:
2513       /* Has the classifier flagged this buffer for special treatment? */
2514     if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2515         next0 = hm->next_override;
2516
2517       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2518         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2519         u32 trace_len = (hbh0->length + 1) << 3;
2520         t->next_index = next0;
2521         /* Capture the h-b-h option verbatim */
2522         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2523         t->trace_len = trace_len;
2524         clib_memcpy(t->option_data, hbh0, trace_len);
2525       }
2526
2527       b0->error = error_node->errors[error0];
2528
2529       /* verify speculative enqueue, maybe switch current next frame */
2530       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2531     }
2532     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2533   }
2534   return frame->n_vectors;
2535 }
2536
2537 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2538   .function = ip6_hop_by_hop,
2539   .name = "ip6-hop-by-hop",
2540   .sibling_of = "ip6-lookup",
2541   .vector_size = sizeof (u32),
2542   .format_trace = format_ip6_hop_by_hop_trace,
2543   .type = VLIB_NODE_TYPE_INTERNAL,
2544   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2545   .error_strings = ip6_hop_by_hop_error_strings,
2546   .n_next_nodes = 0,
2547 };
2548
2549 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2550
2551 static clib_error_t *
2552 ip6_hop_by_hop_init (vlib_main_t * vm)
2553 {
2554   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2555   memset(hm->options, 0, sizeof(hm->options));
2556   memset(hm->trace, 0, sizeof(hm->trace));
2557   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2558   return (0);
2559 }
2560
2561 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2562
2563 void ip6_hbh_set_next_override (uword next)
2564 {
2565   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2566
2567   hm->next_override = next;
2568 }
2569
2570 int
2571 ip6_hbh_register_option (u8 option,
2572                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2573                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2574 {
2575   ip6_main_t * im = &ip6_main;
2576   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2577
2578   ASSERT (option < ARRAY_LEN (hm->options));
2579
2580   /* Already registered */
2581   if (hm->options[option])
2582     return (-1);
2583
2584   hm->options[option] = options;
2585   hm->trace[option] = trace;
2586
2587   /* Set global variable */
2588   im->hbh_enabled = 1;
2589
2590   return (0);
2591 }
2592
2593 int
2594 ip6_hbh_unregister_option (u8 option)
2595 {
2596   ip6_main_t * im = &ip6_main;
2597   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2598
2599   ASSERT (option < ARRAY_LEN (hm->options));
2600
2601   /* Not registered */
2602   if (!hm->options[option])
2603     return (-1);
2604
2605   hm->options[option] = NULL;
2606   hm->trace[option] = NULL;
2607
2608   /* Disable global knob if this was the last option configured */
2609   int i;
2610   bool found = false;
2611   for (i = 0; i < 256; i++) {
2612     if (hm->options[option]) {
2613       found = true;
2614       break;
2615     }
2616   }
2617   if (!found)
2618     im->hbh_enabled = 0;
2619
2620   return (0);
2621 }
2622
2623 /* Global IP6 main. */
2624 ip6_main_t ip6_main;
2625
2626 static clib_error_t *
2627 ip6_lookup_init (vlib_main_t * vm)
2628 {
2629   ip6_main_t * im = &ip6_main;
2630   clib_error_t * error;
2631   uword i;
2632
2633   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2634     return error;
2635
2636   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2637     {
2638       u32 j, i0, i1;
2639
2640       i0 = i / 32;
2641       i1 = i % 32;
2642
2643       for (j = 0; j < i0; j++)
2644         im->fib_masks[i].as_u32[j] = ~0;
2645
2646       if (i1)
2647         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2648     }
2649
2650   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2651
2652   if (im->lookup_table_nbuckets == 0)
2653     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2654
2655   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2656
2657   if (im->lookup_table_size == 0)
2658     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2659
2660   BV(clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2661                         "ip6 FIB fwding table",
2662                         im->lookup_table_nbuckets,
2663                         im->lookup_table_size);
2664   BV(clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2665                         "ip6 FIB non-fwding table",
2666                         im->lookup_table_nbuckets,
2667                         im->lookup_table_size);
2668
2669   /* Create FIB with index 0 and table id of 0. */
2670   fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 0);
2671
2672   {
2673     pg_node_t * pn;
2674     pn = pg_get_node (ip6_lookup_node.index);
2675     pn->unformat_edit = unformat_pg_ip6_header;
2676   }
2677
2678   /* Unless explicitly configured, don't process HBH options */
2679   im->hbh_enabled = 0;
2680
2681   {
2682     icmp6_neighbor_solicitation_header_t p;
2683
2684     memset (&p, 0, sizeof (p));
2685
2686     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2687     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2688                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2689     p.ip.protocol = IP_PROTOCOL_ICMP6;
2690     p.ip.hop_limit = 255;
2691     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2692
2693     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2694
2695     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2696     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2697
2698     vlib_packet_template_init (vm,
2699                                &im->discover_neighbor_packet_template,
2700                                &p, sizeof (p),
2701                                /* alloc chunk size */ 8,
2702                                "ip6 neighbor discovery");
2703   }
2704
2705   return error;
2706 }
2707
2708 VLIB_INIT_FUNCTION (ip6_lookup_init);
2709
2710 static clib_error_t *
2711 add_del_ip6_interface_table (vlib_main_t * vm,
2712                              unformat_input_t * input,
2713                              vlib_cli_command_t * cmd)
2714 {
2715   vnet_main_t * vnm = vnet_get_main();
2716   clib_error_t * error = 0;
2717   u32 sw_if_index, table_id;
2718
2719   sw_if_index = ~0;
2720
2721   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2722     {
2723       error = clib_error_return (0, "unknown interface `%U'",
2724                                  format_unformat_error, input);
2725       goto done;
2726     }
2727
2728   if (unformat (input, "%d", &table_id))
2729     ;
2730   else
2731     {
2732       error = clib_error_return (0, "expected table id `%U'",
2733                                  format_unformat_error, input);
2734       goto done;
2735     }
2736
2737   {
2738     u32 fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6,
2739                                                       table_id);
2740
2741     vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
2742     ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
2743   }
2744
2745
2746  done:
2747   return error;
2748 }
2749
2750 /*?
2751  * Place the indicated interface into the supplied IPv6 FIB table (also known
2752  * as a VRF). If the FIB table does not exist, this command creates it. To
2753  * display the current IPv6 FIB table, use the command '<em>show ip6 fib</em>'.
2754  * FIB table will only be displayed if a route has been added to the table, or
2755  * an IP Address is assigned to an interface in the table (which adds a route
2756  * automatically).
2757  *
2758  * @note IP addresses added after setting the interface IP table end up in
2759  * the indicated FIB table. If the IP address is added prior to adding the
2760  * interface to the FIB table, it will NOT be part of the FIB table. Predictable
2761  * but potentially counter-intuitive results occur if you provision interface
2762  * addresses in multiple FIBs. Upon RX, packets will be processed in the last
2763  * IP table ID provisioned. It might be marginally useful to evade source RPF
2764  * drops to put an interface address into multiple FIBs.
2765  *
2766  * @cliexpar
2767  * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id):
2768  * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2}
2769  ?*/
2770 /* *INDENT-OFF* */
2771 VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) = {
2772   .path = "set interface ip6 table",
2773   .function = add_del_ip6_interface_table,
2774   .short_help = "set interface ip6 table <interface> <table-id>"
2775 };
2776 /* *INDENT-ON* */
2777
2778 void
2779 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
2780                                                   u8 *mac)
2781 {
2782   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2783   /* Invert the "u" bit */
2784   ip->as_u8 [8] = mac[0] ^ (1<<1);
2785   ip->as_u8 [9] = mac[1];
2786   ip->as_u8 [10] = mac[2];
2787   ip->as_u8 [11] = 0xFF;
2788   ip->as_u8 [12] = 0xFE;
2789   ip->as_u8 [13] = mac[3];
2790   ip->as_u8 [14] = mac[4];
2791   ip->as_u8 [15] = mac[5];
2792 }
2793
2794 void
2795 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
2796                                                   ip6_address_t *ip)
2797 {
2798   /* Invert the previously inverted "u" bit */
2799   mac[0] = ip->as_u8 [8] ^ (1<<1);
2800   mac[1] = ip->as_u8 [9];
2801   mac[2] = ip->as_u8 [10];
2802   mac[3] = ip->as_u8 [13];
2803   mac[4] = ip->as_u8 [14];
2804   mac[5] = ip->as_u8 [15];
2805 }
2806
2807 static clib_error_t *
2808 test_ip6_link_command_fn (vlib_main_t * vm,
2809                           unformat_input_t * input,
2810                           vlib_cli_command_t * cmd)
2811 {
2812   u8 mac[6];
2813   ip6_address_t _a, *a = &_a;
2814
2815   if (unformat (input, "%U", unformat_ethernet_address, mac))
2816     {
2817       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2818       vlib_cli_output (vm, "Link local address: %U",
2819                        format_ip6_address, a);
2820       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2821       vlib_cli_output (vm, "Original MAC address: %U",
2822                        format_ethernet_address, mac);
2823     }
2824
2825   return 0;
2826 }
2827
2828 /*?
2829  * This command converts the given MAC Address into an IPv6 link-local
2830  * address.
2831  *
2832  * @cliexpar
2833  * Example of how to create an IPv6 link-local address:
2834  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2835  * Link local address: fe80::14d9:e0ff:fe91:7986
2836  * Original MAC address: 16:d9:e0:91:79:86
2837  * @cliexend
2838 ?*/
2839 /* *INDENT-OFF* */
2840 VLIB_CLI_COMMAND (test_link_command, static) = {
2841   .path = "test ip6 link",
2842   .function = test_ip6_link_command_fn,
2843   .short_help = "test ip6 link <mac-address>",
2844 };
2845 /* *INDENT-ON* */
2846
2847 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2848 {
2849   ip6_main_t * im6 = &ip6_main;
2850   ip6_fib_t * fib;
2851   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
2852
2853   if (p == 0)
2854     return -1;
2855
2856   fib = ip6_fib_get (p[0]);
2857
2858   fib->flow_hash_config = flow_hash_config;
2859   return 1;
2860 }
2861
2862 static clib_error_t *
2863 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2864                               unformat_input_t * input,
2865                               vlib_cli_command_t * cmd)
2866 {
2867   int matched = 0;
2868   u32 table_id = 0;
2869   u32 flow_hash_config = 0;
2870   int rv;
2871
2872   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2873     if (unformat (input, "table %d", &table_id))
2874       matched = 1;
2875 #define _(a,v) \
2876     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2877     foreach_flow_hash_bit
2878 #undef _
2879     else break;
2880   }
2881
2882   if (matched == 0)
2883     return clib_error_return (0, "unknown input `%U'",
2884                               format_unformat_error, input);
2885
2886   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2887   switch (rv)
2888     {
2889     case 1:
2890       break;
2891
2892     case -1:
2893       return clib_error_return (0, "no such FIB table %d", table_id);
2894
2895     default:
2896       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2897       break;
2898     }
2899
2900   return 0;
2901 }
2902
2903 /*?
2904  * Configure the set of IPv6 fields used by the flow hash.
2905  *
2906  * @cliexpar
2907  * @parblock
2908  * Example of how to set the flow hash on a given table:
2909  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2910  *
2911  * Example of display the configured flow hash:
2912  * @cliexstart{show ip6 fib}
2913  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2914  * @::/0
2915  *   unicast-ip6-chain
2916  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2917  *     [0] [@0]: dpo-drop ip6
2918  * fe80::/10
2919  *   unicast-ip6-chain
2920  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2921  *     [0] [@2]: dpo-receive
2922  * ff02::1/128
2923  *   unicast-ip6-chain
2924  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2925  *     [0] [@2]: dpo-receive
2926  * ff02::2/128
2927  *   unicast-ip6-chain
2928  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2929  *     [0] [@2]: dpo-receive
2930  * ff02::16/128
2931  *   unicast-ip6-chain
2932  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2933  *     [0] [@2]: dpo-receive
2934  * ff02::1:ff00:0/104
2935  *   unicast-ip6-chain
2936  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2937  *     [0] [@2]: dpo-receive
2938  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2939  * @::/0
2940  *   unicast-ip6-chain
2941  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2942  *     [0] [@0]: dpo-drop ip6
2943  * @::a:1:1:0:4/126
2944  *   unicast-ip6-chain
2945  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2946  *     [0] [@4]: ipv6-glean: af_packet0
2947  * @::a:1:1:0:7/128
2948  *   unicast-ip6-chain
2949  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2950  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2951  * fe80::/10
2952  *   unicast-ip6-chain
2953  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2954  *     [0] [@2]: dpo-receive
2955  * fe80::fe:3eff:fe3e:9222/128
2956  *   unicast-ip6-chain
2957  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2958  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2959  * ff02::1/128
2960  *   unicast-ip6-chain
2961  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2962  *     [0] [@2]: dpo-receive
2963  * ff02::2/128
2964  *   unicast-ip6-chain
2965  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2966  *     [0] [@2]: dpo-receive
2967  * ff02::16/128
2968  *   unicast-ip6-chain
2969  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2970  *     [0] [@2]: dpo-receive
2971  * ff02::1:ff00:0/104
2972  *   unicast-ip6-chain
2973  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2974  *     [0] [@2]: dpo-receive
2975  * @cliexend
2976  * @endparblock
2977 ?*/
2978 /* *INDENT-OFF* */
2979 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
2980     .path = "set ip6 flow-hash",
2981     .short_help =
2982     "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2983     .function = set_ip6_flow_hash_command_fn,
2984 };
2985 /* *INDENT-ON* */
2986
2987 static clib_error_t *
2988 show_ip6_local_command_fn (vlib_main_t * vm,
2989                            unformat_input_t * input,
2990                            vlib_cli_command_t * cmd)
2991 {
2992   ip6_main_t * im = &ip6_main;
2993   ip_lookup_main_t * lm = &im->lookup_main;
2994   int i;
2995
2996   vlib_cli_output (vm, "Protocols handled by ip6_local");
2997   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2998     {
2999       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3000         vlib_cli_output (vm, "%d", i);
3001     }
3002   return 0;
3003 }
3004
3005
3006
3007 /*?
3008  * Display the set of protocols handled by the local IPv6 stack.
3009  *
3010  * @cliexpar
3011  * Example of how to display local protocol table:
3012  * @cliexstart{show ip6 local}
3013  * Protocols handled by ip6_local
3014  * 17
3015  * 43
3016  * 58
3017  * 115
3018  * @cliexend
3019 ?*/
3020 /* *INDENT-OFF* */
3021 VLIB_CLI_COMMAND (show_ip6_local, static) = {
3022   .path = "show ip6 local",
3023   .function = show_ip6_local_command_fn,
3024   .short_help = "show ip6 local",
3025 };
3026 /* *INDENT-ON* */
3027
3028 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3029                                  u32 table_index)
3030 {
3031   vnet_main_t * vnm = vnet_get_main();
3032   vnet_interface_main_t * im = &vnm->interface_main;
3033   ip6_main_t * ipm = &ip6_main;
3034   ip_lookup_main_t * lm = &ipm->lookup_main;
3035   vnet_classify_main_t * cm = &vnet_classify_main;
3036   ip6_address_t *if_addr;
3037
3038   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3039     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3040
3041   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3042     return VNET_API_ERROR_NO_SUCH_ENTRY;
3043
3044   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3045   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3046
3047   if_addr = ip6_interface_first_address (ipm, sw_if_index, NULL);
3048
3049   if (NULL != if_addr)
3050   {
3051       fib_prefix_t pfx = {
3052           .fp_len = 128,
3053           .fp_proto = FIB_PROTOCOL_IP6,
3054           .fp_addr.ip6 = *if_addr,
3055       };
3056       u32 fib_index;
3057
3058       fib_index = fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
3059                                                       sw_if_index);
3060
3061
3062       if (table_index != (u32) ~0)
3063       {
3064           dpo_id_t dpo = DPO_INVALID;
3065
3066           dpo_set(&dpo,
3067                   DPO_CLASSIFY,
3068                   DPO_PROTO_IP6,
3069                   classify_dpo_create(DPO_PROTO_IP6,
3070                                       table_index));
3071
3072           fib_table_entry_special_dpo_add(fib_index,
3073                                           &pfx,
3074                                           FIB_SOURCE_CLASSIFY,
3075                                           FIB_ENTRY_FLAG_NONE,
3076                                           &dpo);
3077           dpo_reset(&dpo);
3078       }
3079       else
3080       {
3081           fib_table_entry_special_remove(fib_index,
3082                                          &pfx,
3083                                          FIB_SOURCE_CLASSIFY);
3084       }
3085   }
3086
3087   return 0;
3088 }
3089
3090 static clib_error_t *
3091 set_ip6_classify_command_fn (vlib_main_t * vm,
3092                              unformat_input_t * input,
3093                              vlib_cli_command_t * cmd)
3094 {
3095   u32 table_index = ~0;
3096   int table_index_set = 0;
3097   u32 sw_if_index = ~0;
3098   int rv;
3099
3100   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3101     if (unformat (input, "table-index %d", &table_index))
3102       table_index_set = 1;
3103     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3104                        vnet_get_main(), &sw_if_index))
3105         ;
3106     else
3107         break;
3108   }
3109
3110   if (table_index_set == 0)
3111       return clib_error_return (0, "classify table-index must be specified");
3112
3113   if (sw_if_index == ~0)
3114     return clib_error_return (0, "interface / subif must be specified");
3115
3116   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3117
3118   switch (rv)
3119     {
3120     case 0:
3121       break;
3122
3123     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3124       return clib_error_return (0, "No such interface");
3125
3126     case VNET_API_ERROR_NO_SUCH_ENTRY:
3127       return clib_error_return (0, "No such classifier table");
3128     }
3129   return 0;
3130 }
3131
3132 /*?
3133  * Assign a classification table to an interface. The classification
3134  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3135  * commands. Once the table is create, use this command to filter packets
3136  * on an interface.
3137  *
3138  * @cliexpar
3139  * Example of how to assign a classification table to an interface:
3140  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3141 ?*/
3142 /* *INDENT-OFF* */
3143 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3144     .path = "set ip6 classify",
3145     .short_help =
3146     "set ip6 classify intfc <interface> table-index <classify-idx>",
3147     .function = set_ip6_classify_command_fn,
3148 };
3149 /* *INDENT-ON* */
3150
3151 static clib_error_t *
3152 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3153 {
3154   ip6_main_t * im = &ip6_main;
3155   uword heapsize = 0;
3156   u32 tmp;
3157   u32 nbuckets = 0;
3158
3159   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3160     if (unformat (input, "hash-buckets %d", &tmp))
3161       nbuckets = tmp;
3162     else if (unformat (input, "heap-size %dm", &tmp))
3163       heapsize = ((u64)tmp) << 20;
3164     else if (unformat (input, "heap-size %dM", &tmp))
3165       heapsize = ((u64)tmp) << 20;
3166     else if (unformat (input, "heap-size %dg", &tmp))
3167       heapsize = ((u64)tmp) << 30;
3168     else if (unformat (input, "heap-size %dG", &tmp))
3169       heapsize = ((u64)tmp) << 30;
3170     else
3171       return clib_error_return (0, "unknown input '%U'",
3172                                 format_unformat_error, input);
3173   }
3174
3175   im->lookup_table_nbuckets = nbuckets;
3176   im->lookup_table_size = heapsize;
3177
3178   return 0;
3179 }
3180
3181 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");