vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45 #include <vnet/fib/fib_table.h>
  46 #include <vnet/fib/ip6_fib.h>
  47 #include <vnet/dpo/load_balance.h>
  48 #include <vnet/dpo/classify_dpo.h>
  49
  50 #include <vppinfra/bihash_template.c>
  51
  52 /**
  53  * @file
  54  * @brief IPv6 Forwarding.
  55  *
  56  * This file contains the source code for IPv6 forwarding.
  57  */
  58
  59 void
  60 ip6_forward_next_trace (vlib_main_t * vm,
  61                         vlib_node_runtime_t * node,
  62                         vlib_frame_t * frame,
  63                         vlib_rx_or_tx_t which_adj_index);
  64
  65 always_inline uword
  66 ip6_lookup_inline (vlib_main_t * vm,
  67                    vlib_node_runtime_t * node,
  68                    vlib_frame_t * frame)
  69 {
  70   ip6_main_t * im = &ip6_main;
  71   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  72   u32 n_left_from, n_left_to_next, * from, * to_next;
  73   ip_lookup_next_t next;
  74   u32 cpu_index = os_get_cpu_number();
  75
  76   from = vlib_frame_vector_args (frame);
  77   n_left_from = frame->n_vectors;
  78   next = node->cached_next_index;
  79
  80   while (n_left_from > 0)
  81     {
  82       vlib_get_next_frame (vm, node, next,
  83                            to_next, n_left_to_next);
  84
  85       while (n_left_from >= 4 && n_left_to_next >= 2)
  86         {
  87           vlib_buffer_t * p0, * p1;
  88           u32 pi0, pi1, lbi0, lbi1, wrong_next;
  89           ip_lookup_next_t next0, next1;
  90           ip6_header_t * ip0, * ip1;
  91           ip6_address_t * dst_addr0, * dst_addr1;
  92           u32 fib_index0, fib_index1;
  93           u32 flow_hash_config0, flow_hash_config1;
  94           const dpo_id_t *dpo0, *dpo1;
  95           const load_balance_t *lb0, *lb1;
  96
  97           /* Prefetch next iteration. */
  98           {
  99             vlib_buffer_t * p2, * p3;
 100
 101             p2 = vlib_get_buffer (vm, from[2]);
 102             p3 = vlib_get_buffer (vm, from[3]);
 103
 104             vlib_prefetch_buffer_header (p2, LOAD);
 105             vlib_prefetch_buffer_header (p3, LOAD);
 106             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 107             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 108           }
 109
 110           pi0 = to_next[0] = from[0];
 111           pi1 = to_next[1] = from[1];
 112
 113           p0 = vlib_get_buffer (vm, pi0);
 114           p1 = vlib_get_buffer (vm, pi1);
 115
 116           ip0 = vlib_buffer_get_current (p0);
 117           ip1 = vlib_buffer_get_current (p1);
 118
 119           dst_addr0 = &ip0->dst_address;
 120           dst_addr1 = &ip1->dst_address;
 121
 122           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 123           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 124
 125           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 126             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 127           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 128             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 129
 130           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 131           lbi1 = ip6_fib_table_fwding_lookup (im, fib_index1, dst_addr1);
 132
 133           lb0 = load_balance_get (lbi0);
 134           lb1 = load_balance_get (lbi1);
 135
 136           vnet_buffer (p0)->ip.flow_hash =
 137             vnet_buffer(p1)->ip.flow_hash = 0;
 138
 139           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 140             {
 141               flow_hash_config0 = lb0->lb_hash_config;
 142               vnet_buffer (p0)->ip.flow_hash =
 143                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 144             }
 145           if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
 146             {
 147               flow_hash_config1 = lb1->lb_hash_config;
 148               vnet_buffer (p1)->ip.flow_hash =
 149                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 150             }
 151
 152           ASSERT (lb0->lb_n_buckets > 0);
 153           ASSERT (lb1->lb_n_buckets > 0);
 154           ASSERT (is_pow2 (lb0->lb_n_buckets));
 155           ASSERT (is_pow2 (lb1->lb_n_buckets));
 156           dpo0 = load_balance_get_bucket_i(lb0,
 157                                            (vnet_buffer (p0)->ip.flow_hash &
 158                                             lb0->lb_n_buckets_minus_1));
 159           dpo1 = load_balance_get_bucket_i(lb1,
 160                                            (vnet_buffer (p1)->ip.flow_hash &
 161                                             lb1->lb_n_buckets_minus_1));
 162
 163           next0 = dpo0->dpoi_next_node;
 164           next1 = dpo1->dpoi_next_node;
 165
 166           /* Only process the HBH Option Header if explicitly configured to do so */
 167           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 168             {
 169               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 170                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 171             }
 172           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 173             {
 174               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 175                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 176             }
 177           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 178           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 179
 180           vlib_increment_combined_counter
 181               (cm, cpu_index, lbi0, 1,
 182                vlib_buffer_length_in_chain (vm, p0));
 183           vlib_increment_combined_counter
 184               (cm, cpu_index, lbi1, 1,
 185                vlib_buffer_length_in_chain (vm, p1));
 186
 187           from += 2;
 188           to_next += 2;
 189           n_left_to_next -= 2;
 190           n_left_from -= 2;
 191
 192           wrong_next = (next0 != next) + 2*(next1 != next);
 193           if (PREDICT_FALSE (wrong_next != 0))
 194             {
 195               switch (wrong_next)
 196                 {
 197                 case 1:
 198                   /* A B A */
 199                   to_next[-2] = pi1;
 200                   to_next -= 1;
 201                   n_left_to_next += 1;
 202                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 203                   break;
 204
 205                 case 2:
 206                   /* A A B */
 207                   to_next -= 1;
 208                   n_left_to_next += 1;
 209                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 210                   break;
 211
 212                 case 3:
 213                   /* A B C */
 214                   to_next -= 2;
 215                   n_left_to_next += 2;
 216                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 217                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 218                   if (next0 == next1)
 219                     {
 220                       /* A B B */
 221                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 222                       next = next1;
 223                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 224                     }
 225                 }
 226             }
 227         }
 228
 229       while (n_left_from > 0 && n_left_to_next > 0)
 230         {
 231           vlib_buffer_t * p0;
 232           ip6_header_t * ip0;
 233           u32 pi0, lbi0;
 234           ip_lookup_next_t next0;
 235           load_balance_t * lb0;
 236           ip6_address_t * dst_addr0;
 237           u32 fib_index0, flow_hash_config0;
 238           const dpo_id_t *dpo0;
 239
 240           pi0 = from[0];
 241           to_next[0] = pi0;
 242
 243           p0 = vlib_get_buffer (vm, pi0);
 244
 245           ip0 = vlib_buffer_get_current (p0);
 246
 247           dst_addr0 = &ip0->dst_address;
 248
 249           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 250           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 251             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 252
 253           flow_hash_config0 =
 254               ip6_fib_get (fib_index0)->flow_hash_config;
 255
 256           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 257
 258           lb0 = load_balance_get (lbi0);
 259
 260           vnet_buffer (p0)->ip.flow_hash = 0;
 261
 262           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 263             {
 264               flow_hash_config0 = lb0->lb_hash_config;
 265               vnet_buffer (p0)->ip.flow_hash =
 266                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 267             }
 268
 269           ASSERT (lb0->lb_n_buckets > 0);
 270           ASSERT (is_pow2 (lb0->lb_n_buckets));
 271           dpo0 = load_balance_get_bucket_i(lb0,
 272                                            (vnet_buffer (p0)->ip.flow_hash &
 273                                             lb0->lb_n_buckets_minus_1));
 274           next0 = dpo0->dpoi_next_node;
 275
 276           /* Only process the HBH Option Header if explicitly configured to do so */
 277           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 278             {
 279               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 280                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 281             }
 282           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 283
 284           vlib_increment_combined_counter
 285               (cm, cpu_index, lbi0, 1,
 286                vlib_buffer_length_in_chain (vm, p0));
 287
 288           from += 1;
 289           to_next += 1;
 290           n_left_to_next -= 1;
 291           n_left_from -= 1;
 292
 293           if (PREDICT_FALSE (next0 != next))
 294             {
 295               n_left_to_next += 1;
 296               vlib_put_next_frame (vm, node, next, n_left_to_next);
 297               next = next0;
 298               vlib_get_next_frame (vm, node, next,
 299                                    to_next, n_left_to_next);
 300               to_next[0] = pi0;
 301               to_next += 1;
 302               n_left_to_next -= 1;
 303             }
 304         }
 305
 306       vlib_put_next_frame (vm, node, next, n_left_to_next);
 307     }
 308
 309   if (node->flags & VLIB_NODE_FLAG_TRACE)
 310     ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 311
 312   return frame->n_vectors;
 313 }
 314
 315 static void
 316 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 317                           ip6_main_t * im, u32 fib_index,
 318                           ip_interface_address_t * a)
 319 {
 320   ip_lookup_main_t * lm = &im->lookup_main;
 321   ip6_address_t * address = ip_interface_address_get_address (lm, a);
 322   fib_prefix_t pfx = {
 323       .fp_len = a->address_length,
 324       .fp_proto = FIB_PROTOCOL_IP6,
 325       .fp_addr.ip6 = *address,
 326   };
 327
 328   a->neighbor_probe_adj_index = ~0;
 329   if (a->address_length < 128)
 330   {
 331       fib_node_index_t fei;
 332
 333       fei = fib_table_entry_update_one_path(fib_index,
 334                                             &pfx,
 335                                             FIB_SOURCE_INTERFACE,
 336                                             (FIB_ENTRY_FLAG_CONNECTED |
 337                                              FIB_ENTRY_FLAG_ATTACHED),
 338                                             FIB_PROTOCOL_IP6,
 339                                             NULL, /* No next-hop address */
 340                                             sw_if_index,
 341                                             ~0, // invalid FIB index
 342                                             1,
 343                                             NULL, // no label stack
 344                                             FIB_ROUTE_PATH_FLAG_NONE);
 345       a->neighbor_probe_adj_index = fib_entry_get_adj(fei);
 346   }
 347
 348   pfx.fp_len = 128;
 349   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 350   {
 351       u32 classify_table_index =
 352           lm->classify_table_index_by_sw_if_index [sw_if_index];
 353       if (classify_table_index != (u32) ~0)
 354       {
 355           dpo_id_t dpo = DPO_INVALID;
 356
 357           dpo_set(&dpo,
 358                   DPO_CLASSIFY,
 359                   DPO_PROTO_IP6,
 360                   classify_dpo_create(DPO_PROTO_IP6, classify_table_index));
 361
 362           fib_table_entry_special_dpo_add(fib_index,
 363                                           &pfx,
 364                                           FIB_SOURCE_CLASSIFY,
 365                                           FIB_ENTRY_FLAG_NONE,
 366                                           &dpo);
 367           dpo_reset(&dpo);
 368       }
 369   }
 370
 371   fib_table_entry_update_one_path(fib_index,
 372                                   &pfx,
 373                                   FIB_SOURCE_INTERFACE,
 374                                   (FIB_ENTRY_FLAG_CONNECTED |
 375                                    FIB_ENTRY_FLAG_LOCAL),
 376                                   FIB_PROTOCOL_IP6,
 377                                   &pfx.fp_addr,
 378                                   sw_if_index,
 379                                   ~0, // invalid FIB index
 380                                   1,
 381                                   NULL,
 382                                   FIB_ROUTE_PATH_FLAG_NONE);
 383 }
 384
 385 static void
 386 ip6_del_interface_routes (ip6_main_t * im,
 387                           u32 fib_index,
 388                           ip6_address_t * address,
 389                           u32 address_length)
 390 {
 391     fib_prefix_t pfx = {
 392         .fp_len = address_length,
 393         .fp_proto = FIB_PROTOCOL_IP6,
 394         .fp_addr.ip6 = *address,
 395     };
 396
 397     if (pfx.fp_len < 128)
 398     {
 399         fib_table_entry_delete(fib_index,
 400                                &pfx,
 401                                FIB_SOURCE_INTERFACE);
 402
 403     }
 404
 405     pfx.fp_len = 128;
 406     fib_table_entry_delete(fib_index,
 407                            &pfx,
 408                            FIB_SOURCE_INTERFACE);
 409 }
 410
 411 void
 412 ip6_sw_interface_enable_disable (u32 sw_if_index,
 413                                  u32 is_enable)
 414 {
 415   ip6_main_t * im = &ip6_main;
 416
 417   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 418
 419   /*
 420    * enable/disable only on the 1<->0 transition
 421    */
 422   if (is_enable)
 423     {
 424       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 425         return;
 426     }
 427   else
 428     {
 429       ASSERT(im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 430       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 431         return;
 432     }
 433
 434   vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index,
 435                                is_enable, 0, 0);
 436
 437   vnet_feature_enable_disable ("ip6-multicast", "ip6-lookup", sw_if_index,
 438                                is_enable, 0, 0);
 439
 440 }
 441
 442 /* get first interface address */
 443 ip6_address_t *
 444 ip6_interface_first_address (ip6_main_t * im,
 445                              u32 sw_if_index,
 446                              ip_interface_address_t ** result_ia)
 447 {
 448   ip_lookup_main_t * lm = &im->lookup_main;
 449   ip_interface_address_t * ia = 0;
 450   ip6_address_t * result = 0;
 451
 452   foreach_ip_interface_address (lm, ia, sw_if_index,
 453                                 1 /* honor unnumbered */,
 454   ({
 455     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 456     result = a;
 457     break;
 458   }));
 459   if (result_ia)
 460     *result_ia = result ? ia : 0;
 461   return result;
 462 }
 463
 464 clib_error_t *
 465 ip6_add_del_interface_address (vlib_main_t * vm,
 466                                u32 sw_if_index,
 467                                ip6_address_t * address,
 468                                u32 address_length,
 469                                u32 is_del)
 470 {
 471   vnet_main_t * vnm = vnet_get_main();
 472   ip6_main_t * im = &ip6_main;
 473   ip_lookup_main_t * lm = &im->lookup_main;
 474   clib_error_t * error;
 475   u32 if_address_index;
 476   ip6_address_fib_t ip6_af, * addr_fib = 0;
 477
 478   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 479   ip6_addr_fib_init (&ip6_af, address,
 480                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 481   vec_add1 (addr_fib, ip6_af);
 482
 483   {
 484     uword elts_before = pool_elts (lm->if_address_pool);
 485
 486     error = ip_interface_address_add_del
 487       (lm,
 488        sw_if_index,
 489        addr_fib,
 490        address_length,
 491        is_del,
 492        &if_address_index);
 493     if (error)
 494       goto done;
 495
 496     /* Pool did not grow: add duplicate address. */
 497     if (elts_before == pool_elts (lm->if_address_pool))
 498       goto done;
 499   }
 500
 501   ip6_sw_interface_enable_disable(sw_if_index, !is_del);
 502
 503   if (is_del)
 504       ip6_del_interface_routes (im, ip6_af.fib_index, address,
 505                                 address_length);
 506   else
 507       ip6_add_interface_routes (vnm, sw_if_index,
 508                                 im, ip6_af.fib_index,
 509                                 pool_elt_at_index (lm->if_address_pool, if_address_index));
 510
 511   {
 512     ip6_add_del_interface_address_callback_t * cb;
 513     vec_foreach (cb, im->add_del_interface_address_callbacks)
 514       cb->function (im, cb->function_opaque, sw_if_index,
 515                     address, address_length,
 516                     if_address_index,
 517                     is_del);
 518   }
 519
 520  done:
 521   vec_free (addr_fib);
 522   return error;
 523 }
 524
 525 clib_error_t *
 526 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
 527                                 u32 sw_if_index,
 528                                 u32 flags)
 529 {
 530   ip6_main_t * im = &ip6_main;
 531   ip_interface_address_t * ia;
 532   ip6_address_t * a;
 533   u32 is_admin_up, fib_index;
 534
 535   /* Fill in lookup tables with default table (0). */
 536   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 537
 538   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
 539
 540   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 541
 542   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 543
 544   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 545                                 0 /* honor unnumbered */,
 546   ({
 547     a = ip_interface_address_get_address (&im->lookup_main, ia);
 548     if (is_admin_up)
 549       ip6_add_interface_routes (vnm, sw_if_index,
 550                                 im, fib_index,
 551                                 ia);
 552     else
 553       ip6_del_interface_routes (im, fib_index,
 554                                 a, ia->address_length);
 555   }));
 556
 557   return 0;
 558 }
 559
 560 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 561
 562 /* Built-in ip6 unicast rx feature path definition */
 563 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 564 {
 565   .arc_name  = "ip6-unicast",
 566   .start_nodes = VNET_FEATURES ("ip6-input"),
 567   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 568 };
 569
 570 VNET_FEATURE_INIT (ip6_flow_classify, static) = {
 571   .arc_name = "ip6-unicast",
 572   .node_name = "ip6-flow-classify",
 573   .runs_before = VNET_FEATURES ("ip6-inacl"),
 574 };
 575
 576 VNET_FEATURE_INIT (ip6_inacl, static) = {
 577   .arc_name = "ip6-unicast",
 578   .node_name = "ip6-inacl",
 579   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 580 };
 581
 582 VNET_FEATURE_INIT (ip6_policer_classify, static) = {
 583   .arc_name = "ip6-unicast",
 584   .node_name = "ip6-policer-classify",
 585   .runs_before = VNET_FEATURES ("ipsec-input-ip6"),
 586 };
 587
 588 VNET_FEATURE_INIT (ip6_ipsec, static) = {
 589   .arc_name = "ip6-unicast",
 590   .node_name = "ipsec-input-ip6",
 591   .runs_before = VNET_FEATURES ("l2tp-decap"),
 592 };
 593
 594 VNET_FEATURE_INIT (ip6_l2tp, static) = {
 595   .arc_name = "ip6-unicast",
 596   .node_name = "l2tp-decap",
 597   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 598 };
 599
 600 VNET_FEATURE_INIT (ip6_vpath, static) = {
 601   .arc_name = "ip6-unicast",
 602   .node_name = "vpath-input-ip6",
 603   .runs_before = VNET_FEATURES ("ip6-lookup"),
 604 };
 605
 606 VNET_FEATURE_INIT (ip6_lookup, static) = {
 607   .arc_name = "ip6-unicast",
 608   .node_name = "ip6-lookup",
 609   .runs_before = VNET_FEATURES ("ip6-drop"),
 610 };
 611
 612 VNET_FEATURE_INIT (ip6_drop, static) = {
 613   .arc_name = "ip6-unicast",
 614   .node_name = "ip6-drop",
 615   .runs_before = 0,  /*last feature*/
 616 };
 617
 618 /* Built-in ip6 multicast rx feature path definition (none now) */
 619 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 620 {
 621   .arc_name  = "ip6-multicast",
 622   .start_nodes = VNET_FEATURES ("ip6-input"),
 623   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 624 };
 625
 626 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 627   .arc_name = "ip6-multicast",
 628   .node_name = "vpath-input-ip6",
 629   .runs_before = VNET_FEATURES ("ip6-lookup"),
 630 };
 631
 632 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 633   .arc_name = "ip6-multicast",
 634   .node_name = "ip6-lookup",
 635   .runs_before = VNET_FEATURES ("ip6-drop"),
 636 };
 637
 638 VNET_FEATURE_INIT (ip6_drop_mc, static) = {
 639   .arc_name = "ip6-multicast",
 640   .node_name = "ip6-drop",
 641   .runs_before = 0, /* last feature */
 642 };
 643
 644 /* Built-in ip4 tx feature path definition */
 645 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 646 {
 647   .arc_name  = "ip6-output",
 648   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain"),
 649   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 650 };
 651
 652 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 653   .arc_name = "ip6-output",
 654   .node_name = "ipsec-output-ip6",
 655   .runs_before = VNET_FEATURES ("interface-output"),
 656 };
 657
 658 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 659   .arc_name = "ip6-output",
 660   .node_name = "interface-output",
 661   .runs_before = 0, /* not before any other features */
 662 };
 663
 664 clib_error_t *
 665 ip6_sw_interface_add_del (vnet_main_t * vnm,
 666                           u32 sw_if_index,
 667                           u32 is_add)
 668 {
 669   vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index,
 670                                is_add, 0, 0);
 671
 672   vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index,
 673                                is_add, 0, 0);
 674
 675   vnet_feature_enable_disable ("ip6-output", "interface-output", sw_if_index,
 676                                is_add, 0, 0);
 677
 678   return /* no error */ 0;
 679 }
 680
 681 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 682
 683 static uword
 684 ip6_lookup (vlib_main_t * vm,
 685             vlib_node_runtime_t * node,
 686             vlib_frame_t * frame)
 687 {
 688   return ip6_lookup_inline (vm, node, frame);
 689 }
 690
 691 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
 692
 693 VLIB_REGISTER_NODE (ip6_lookup_node) = {
 694   .function = ip6_lookup,
 695   .name = "ip6-lookup",
 696   .vector_size = sizeof (u32),
 697
 698   .format_trace = format_ip6_lookup_trace,
 699
 700   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 701   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 702 };
 703
 704 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
 705
 706 always_inline uword
 707 ip6_load_balance (vlib_main_t * vm,
 708                   vlib_node_runtime_t * node,
 709                   vlib_frame_t * frame)
 710 {
 711   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
 712   u32 n_left_from, n_left_to_next, * from, * to_next;
 713   ip_lookup_next_t next;
 714   u32 cpu_index = os_get_cpu_number();
 715   ip6_main_t * im = &ip6_main;
 716
 717   from = vlib_frame_vector_args (frame);
 718   n_left_from = frame->n_vectors;
 719   next = node->cached_next_index;
 720
 721   if (node->flags & VLIB_NODE_FLAG_TRACE)
 722       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 723
 724   while (n_left_from > 0)
 725     {
 726       vlib_get_next_frame (vm, node, next,
 727                            to_next, n_left_to_next);
 728
 729
 730       while (n_left_from >= 4 && n_left_to_next >= 2)
 731         {
 732           ip_lookup_next_t next0, next1;
 733           const load_balance_t *lb0, *lb1;
 734           vlib_buffer_t * p0, *p1;
 735           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 736           const ip6_header_t *ip0, *ip1;
 737           const dpo_id_t *dpo0, *dpo1;
 738
 739           /* Prefetch next iteration. */
 740           {
 741             vlib_buffer_t * p2, * p3;
 742
 743             p2 = vlib_get_buffer (vm, from[2]);
 744             p3 = vlib_get_buffer (vm, from[3]);
 745
 746             vlib_prefetch_buffer_header (p2, STORE);
 747             vlib_prefetch_buffer_header (p3, STORE);
 748
 749             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 750             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 751           }
 752
 753           pi0 = to_next[0] = from[0];
 754           pi1 = to_next[1] = from[1];
 755
 756           from += 2;
 757           n_left_from -= 2;
 758           to_next += 2;
 759           n_left_to_next -= 2;
 760
 761           p0 = vlib_get_buffer (vm, pi0);
 762           p1 = vlib_get_buffer (vm, pi1);
 763
 764           ip0 = vlib_buffer_get_current (p0);
 765           ip1 = vlib_buffer_get_current (p1);
 766           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 767           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 768
 769           lb0 = load_balance_get(lbi0);
 770           lb1 = load_balance_get(lbi1);
 771
 772           /*
 773            * this node is for via FIBs we can re-use the hash value from the
 774            * to node if present.
 775            * We don't want to use the same hash value at each level in the recursion
 776            * graph as that would lead to polarisation
 777            */
 778           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 779           hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
 780
 781           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 782           {
 783               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 784               {
 785                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 786               }
 787               else
 788               {
 789                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 790               }
 791           }
 792           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 793           {
 794               if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash))
 795               {
 796                   hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1;
 797               }
 798               else
 799               {
 800                   hc1 = vnet_buffer(p1)->ip.flow_hash = ip6_compute_flow_hash(ip1, hc1);
 801               }
 802           }
 803
 804           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 805           dpo1 = load_balance_get_bucket_i(lb1, hc1 & (lb1->lb_n_buckets_minus_1));
 806
 807           next0 = dpo0->dpoi_next_node;
 808           next1 = dpo1->dpoi_next_node;
 809
 810           /* Only process the HBH Option Header if explicitly configured to do so */
 811           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 812             {
 813               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 814                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 815             }
 816           /* Only process the HBH Option Header if explicitly configured to do so */
 817           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 818             {
 819               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 820                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 821             }
 822
 823           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 824           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 825
 826           vlib_increment_combined_counter
 827               (cm, cpu_index, lbi0, 1,
 828                vlib_buffer_length_in_chain (vm, p0));
 829           vlib_increment_combined_counter
 830               (cm, cpu_index, lbi1, 1,
 831                vlib_buffer_length_in_chain (vm, p1));
 832
 833           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 834                                            to_next, n_left_to_next,
 835                                            pi0, pi1, next0, next1);
 836        }
 837
 838       while (n_left_from > 0 && n_left_to_next > 0)
 839         {
 840           ip_lookup_next_t next0;
 841           const load_balance_t *lb0;
 842           vlib_buffer_t * p0;
 843           u32 pi0, lbi0, hc0;
 844           const ip6_header_t *ip0;
 845           const dpo_id_t *dpo0;
 846
 847           pi0 = from[0];
 848           to_next[0] = pi0;
 849           from += 1;
 850           to_next += 1;
 851           n_left_to_next -= 1;
 852           n_left_from -= 1;
 853
 854           p0 = vlib_get_buffer (vm, pi0);
 855
 856           ip0 = vlib_buffer_get_current (p0);
 857           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 858
 859           lb0 = load_balance_get(lbi0);
 860
 861           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 862           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 863           {
 864               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 865               {
 866                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 867               }
 868               else
 869               {
 870                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 871               }
 872           }
 873           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 874
 875           next0 = dpo0->dpoi_next_node;
 876           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 877
 878           /* Only process the HBH Option Header if explicitly configured to do so */
 879           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 880             {
 881               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 882                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 883             }
 884
 885           vlib_increment_combined_counter
 886               (cm, cpu_index, lbi0, 1,
 887                vlib_buffer_length_in_chain (vm, p0));
 888
 889           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 890                                            to_next, n_left_to_next,
 891                                            pi0, next0);
 892         }
 893
 894       vlib_put_next_frame (vm, node, next, n_left_to_next);
 895     }
 896
 897   return frame->n_vectors;
 898 }
 899
 900 VLIB_REGISTER_NODE (ip6_load_balance_node) = {
 901   .function = ip6_load_balance,
 902   .name = "ip6-load-balance",
 903   .vector_size = sizeof (u32),
 904   .sibling_of = "ip6-lookup",
 905
 906   .format_trace = format_ip6_lookup_trace,
 907 };
 908
 909 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance)
 910
 911 typedef struct {
 912   /* Adjacency taken. */
 913   u32 adj_index;
 914   u32 flow_hash;
 915   u32 fib_index;
 916
 917   /* Packet data, possibly *after* rewrite. */
 918   u8 packet_data[128 - 1*sizeof(u32)];
 919 } ip6_forward_next_trace_t;
 920
 921 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
 922 {
 923   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 924   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 925   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 926   uword indent = format_get_indent (s);
 927
 928   s = format(s, "%U%U",
 929              format_white_space, indent,
 930              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 931   return s;
 932 }
 933
 934 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
 935 {
 936   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 937   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 938   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 939   uword indent = format_get_indent (s);
 940
 941   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 942               t->fib_index, t->adj_index, t->flow_hash);
 943   s = format(s, "\n%U%U",
 944              format_white_space, indent,
 945              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 946   return s;
 947 }
 948
 949
 950 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
 951 {
 952   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 953   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 954   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 955   vnet_main_t * vnm = vnet_get_main();
 956   uword indent = format_get_indent (s);
 957
 958   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 959               t->fib_index, t->adj_index, format_ip_adjacency,
 960               t->adj_index, FORMAT_IP_ADJACENCY_NONE,
 961               t->flow_hash);
 962   s = format (s, "\n%U%U",
 963               format_white_space, indent,
 964               format_ip_adjacency_packet_data,
 965               vnm, t->adj_index,
 966               t->packet_data, sizeof (t->packet_data));
 967   return s;
 968 }
 969
 970 /* Common trace function for all ip6-forward next nodes. */
 971 void
 972 ip6_forward_next_trace (vlib_main_t * vm,
 973                         vlib_node_runtime_t * node,
 974                         vlib_frame_t * frame,
 975                         vlib_rx_or_tx_t which_adj_index)
 976 {
 977   u32 * from, n_left;
 978   ip6_main_t * im = &ip6_main;
 979
 980   n_left = frame->n_vectors;
 981   from = vlib_frame_vector_args (frame);
 982
 983   while (n_left >= 4)
 984     {
 985       u32 bi0, bi1;
 986       vlib_buffer_t * b0, * b1;
 987       ip6_forward_next_trace_t * t0, * t1;
 988
 989       /* Prefetch next iteration. */
 990       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 991       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 992
 993       bi0 = from[0];
 994       bi1 = from[1];
 995
 996       b0 = vlib_get_buffer (vm, bi0);
 997       b1 = vlib_get_buffer (vm, bi1);
 998
 999       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1000         {
1001           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1002           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1003           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1004           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1005               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1006               vec_elt (im->fib_index_by_sw_if_index,
1007                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1008
1009           clib_memcpy (t0->packet_data,
1010                   vlib_buffer_get_current (b0),
1011                   sizeof (t0->packet_data));
1012         }
1013       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1014         {
1015           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1016           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1017           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1018           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1019               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1020               vec_elt (im->fib_index_by_sw_if_index,
1021                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1022
1023           clib_memcpy (t1->packet_data,
1024                   vlib_buffer_get_current (b1),
1025                   sizeof (t1->packet_data));
1026         }
1027       from += 2;
1028       n_left -= 2;
1029     }
1030
1031   while (n_left >= 1)
1032     {
1033       u32 bi0;
1034       vlib_buffer_t * b0;
1035       ip6_forward_next_trace_t * t0;
1036
1037       bi0 = from[0];
1038
1039       b0 = vlib_get_buffer (vm, bi0);
1040
1041       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1042         {
1043           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1044           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1045           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1046           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1047               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1048               vec_elt (im->fib_index_by_sw_if_index,
1049                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1050
1051           clib_memcpy (t0->packet_data,
1052                   vlib_buffer_get_current (b0),
1053                   sizeof (t0->packet_data));
1054         }
1055       from += 1;
1056       n_left -= 1;
1057     }
1058 }
1059
1060 static uword
1061 ip6_drop_or_punt (vlib_main_t * vm,
1062                   vlib_node_runtime_t * node,
1063                   vlib_frame_t * frame,
1064                   ip6_error_t error_code)
1065 {
1066   u32 * buffers = vlib_frame_vector_args (frame);
1067   uword n_packets = frame->n_vectors;
1068
1069   vlib_error_drop_buffers (vm, node,
1070                            buffers,
1071                            /* stride */ 1,
1072                            n_packets,
1073                            /* next */ 0,
1074                            ip6_input_node.index,
1075                            error_code);
1076
1077   if (node->flags & VLIB_NODE_FLAG_TRACE)
1078     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1079
1080   return n_packets;
1081 }
1082
1083 static uword
1084 ip6_drop (vlib_main_t * vm,
1085           vlib_node_runtime_t * node,
1086           vlib_frame_t * frame)
1087 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1088
1089 static uword
1090 ip6_punt (vlib_main_t * vm,
1091           vlib_node_runtime_t * node,
1092           vlib_frame_t * frame)
1093 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1094
1095 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1096   .function = ip6_drop,
1097   .name = "ip6-drop",
1098   .vector_size = sizeof (u32),
1099
1100   .format_trace = format_ip6_forward_next_trace,
1101
1102   .n_next_nodes = 1,
1103   .next_nodes = {
1104     [0] = "error-drop",
1105   },
1106 };
1107
1108 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1109
1110 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1111   .function = ip6_punt,
1112   .name = "ip6-punt",
1113   .vector_size = sizeof (u32),
1114
1115   .format_trace = format_ip6_forward_next_trace,
1116
1117   .n_next_nodes = 1,
1118   .next_nodes = {
1119     [0] = "error-punt",
1120   },
1121 };
1122
1123 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1124
1125 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1126   .function = ip6_drop,
1127   .name = "ip6-multicast",
1128   .vector_size = sizeof (u32),
1129
1130   .format_trace = format_ip6_forward_next_trace,
1131
1132   .n_next_nodes = 1,
1133   .next_nodes = {
1134     [0] = "error-drop",
1135   },
1136 };
1137
1138 /* Compute TCP/UDP/ICMP6 checksum in software. */
1139 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1140 {
1141   ip_csum_t sum0;
1142   u16 sum16, payload_length_host_byte_order;
1143   u32 i, n_this_buffer, n_bytes_left;
1144   u32 headers_size = sizeof(ip0[0]);
1145   void * data_this_buffer;
1146
1147   ASSERT(bogus_lengthp);
1148   *bogus_lengthp = 0;
1149
1150   /* Initialize checksum with ip header. */
1151   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1152   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1153   data_this_buffer = (void *) (ip0 + 1);
1154
1155   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1156     {
1157       sum0 = ip_csum_with_carry (sum0,
1158                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1159       sum0 = ip_csum_with_carry (sum0,
1160                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1161     }
1162
1163   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1164   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1165     {
1166       u32  skip_bytes;
1167       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1168
1169       /* validate really icmp6 next */
1170       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1171
1172       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1173       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1174
1175       payload_length_host_byte_order  -= skip_bytes;
1176       headers_size += skip_bytes;
1177    }
1178
1179   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1180   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1181     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1182   while (1)
1183     {
1184       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1185       n_bytes_left -= n_this_buffer;
1186       if (n_bytes_left == 0)
1187         break;
1188
1189       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1190         {
1191           *bogus_lengthp = 1;
1192           return 0xfefe;
1193         }
1194       p0 = vlib_get_buffer (vm, p0->next_buffer);
1195       data_this_buffer = vlib_buffer_get_current (p0);
1196       n_this_buffer = p0->current_length;
1197     }
1198
1199   sum16 = ~ ip_csum_fold (sum0);
1200
1201   return sum16;
1202 }
1203
1204 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1205 {
1206   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1207   udp_header_t * udp0;
1208   u16 sum16;
1209   int bogus_length;
1210
1211   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1212   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1213           || ip0->protocol == IP_PROTOCOL_ICMP6
1214           || ip0->protocol == IP_PROTOCOL_UDP
1215           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1216
1217   udp0 = (void *) (ip0 + 1);
1218   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1219     {
1220       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1221                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1222       return p0->flags;
1223     }
1224
1225   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1226
1227   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1228                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1229
1230   return p0->flags;
1231 }
1232
1233 /* ip6_locate_header
1234  *
1235  * This function is to search for the header specified by the find_hdr number.
1236  *   1. If the find_hdr < 0 then it finds and returns the protocol number and
1237  *   offset stored in *offset of the transport or ESP header in the chain if
1238  *   found.
1239  *   2. If a header with find_hdr > 0 protocol number is found then the
1240  *      offset is stored in *offset and protocol number of the header is
1241  *      returned.
1242  *   3. If find_hdr header is not found or packet is malformed or
1243  *      it is a non-first fragment -1 is returned.
1244  */
1245 always_inline int ip6_locate_header (vlib_buffer_t *p0,
1246                                 ip6_header_t *ip0,
1247                                 int find_hdr,
1248                                 u32 *offset)
1249 {
1250   u8 next_proto = ip0->protocol;
1251   u8 *next_header;
1252   u8 done = 0;
1253   u32 cur_offset;
1254   u8  *temp_nxthdr = 0;
1255   u32 exthdr_len = 0;
1256
1257   next_header = ip6_next_header(ip0);
1258   cur_offset = sizeof(ip6_header_t);
1259   while(1)
1260     {
1261       done = (next_proto == find_hdr);
1262       if (PREDICT_FALSE(next_header >= (u8 *)vlib_buffer_get_current(p0) + p0->current_length))
1263         {
1264           //A malicious packet could set an extension header with a too big size
1265           return(-1);
1266         }
1267       if (done)
1268         break;
1269       if ((!ip6_ext_hdr(next_proto)) || next_proto == IP_PROTOCOL_IP6_NONXT)
1270         {
1271           if (find_hdr < 0)
1272             break;
1273           return -1;
1274         }
1275       if (next_proto == IP_PROTOCOL_IPV6_FRAGMENTATION)
1276         {
1277           ip6_frag_hdr_t *frag_hdr = (ip6_frag_hdr_t *)next_header;
1278           u16 frag_off = ip6_frag_hdr_offset(frag_hdr);
1279           /* Non first fragment return -1 */
1280           if (frag_off)
1281             return(-1);
1282           exthdr_len = sizeof(ip6_frag_hdr_t);
1283           temp_nxthdr = next_header + exthdr_len;
1284         }
1285       else if (next_proto == IP_PROTOCOL_IPSEC_AH)
1286         {
1287           exthdr_len = ip6_ext_authhdr_len(((ip6_ext_header_t *)next_header));
1288           temp_nxthdr = next_header + exthdr_len;
1289         }
1290       else
1291         {
1292           exthdr_len = ip6_ext_header_len(((ip6_ext_header_t *)next_header));
1293           temp_nxthdr = next_header + exthdr_len;
1294         }
1295         next_proto = ((ip6_ext_header_t *)next_header)->next_hdr;
1296         next_header = temp_nxthdr;
1297         cur_offset += exthdr_len;
1298     }
1299
1300   *offset = cur_offset;
1301   return(next_proto);
1302 }
1303
1304 static uword
1305 ip6_local (vlib_main_t * vm,
1306            vlib_node_runtime_t * node,
1307            vlib_frame_t * frame)
1308 {
1309   ip6_main_t * im = &ip6_main;
1310   ip_lookup_main_t * lm = &im->lookup_main;
1311   ip_local_next_t next_index;
1312   u32 * from, * to_next, n_left_from, n_left_to_next;
1313   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1314
1315   from = vlib_frame_vector_args (frame);
1316   n_left_from = frame->n_vectors;
1317   next_index = node->cached_next_index;
1318
1319   if (node->flags & VLIB_NODE_FLAG_TRACE)
1320     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1321
1322   while (n_left_from > 0)
1323     {
1324       vlib_get_next_frame (vm, node, next_index,
1325                            to_next, n_left_to_next);
1326
1327       while (n_left_from >= 4 && n_left_to_next >= 2)
1328         {
1329           vlib_buffer_t * p0, * p1;
1330           ip6_header_t * ip0, * ip1;
1331           udp_header_t * udp0, * udp1;
1332           u32 pi0, ip_len0, udp_len0, flags0, next0;
1333           u32 pi1, ip_len1, udp_len1, flags1, next1;
1334           i32 len_diff0, len_diff1;
1335           u8 error0, type0, good_l4_checksum0;
1336           u8 error1, type1, good_l4_checksum1;
1337           u32 udp_offset0, udp_offset1;
1338
1339           pi0 = to_next[0] = from[0];
1340           pi1 = to_next[1] = from[1];
1341           from += 2;
1342           n_left_from -= 2;
1343           to_next += 2;
1344           n_left_to_next -= 2;
1345
1346           p0 = vlib_get_buffer (vm, pi0);
1347           p1 = vlib_get_buffer (vm, pi1);
1348
1349           ip0 = vlib_buffer_get_current (p0);
1350           ip1 = vlib_buffer_get_current (p1);
1351
1352           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1353           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1354
1355           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1356           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1357
1358           flags0 = p0->flags;
1359           flags1 = p1->flags;
1360
1361           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1362           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1363           len_diff0 = 0;
1364           len_diff1 = 0;
1365
1366           /* Skip HBH local processing */
1367           if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1368             {
1369               ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)ip6_next_header(ip0);
1370               next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
1371               type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
1372             }
1373           if (PREDICT_FALSE (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1374             {
1375               ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)ip6_next_header(ip1);
1376               next1 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
1377               type1 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
1378             }
1379           if (PREDICT_TRUE(IP_PROTOCOL_UDP == ip6_locate_header(p0, ip0,
1380                                                            IP_PROTOCOL_UDP, &udp_offset0)))
1381             {
1382               udp0 = (udp_header_t *)((u8 *)ip0 + udp_offset0);
1383               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1384               good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1385               /* Verify UDP length. */
1386               ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1387               udp_len0 = clib_net_to_host_u16 (udp0->length);
1388               len_diff0 = ip_len0 - udp_len0;
1389             }
1390           if (PREDICT_TRUE(IP_PROTOCOL_UDP == ip6_locate_header(p1, ip1,
1391                                                            IP_PROTOCOL_UDP, &udp_offset1)))
1392             {
1393               udp1 = (udp_header_t *)((u8 *)ip1 + udp_offset1);
1394               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1395               good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1396               /* Verify UDP length. */
1397               ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1398               udp_len1 = clib_net_to_host_u16 (udp1->length);
1399               len_diff1 = ip_len1 - udp_len1;
1400             }
1401
1402           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1403           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1404
1405           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1406           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1407
1408           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1409                              && ! good_l4_checksum0
1410                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1411             {
1412               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1413               good_l4_checksum0 =
1414                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1415             }
1416           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1417                              && ! good_l4_checksum1
1418                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1419             {
1420               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1421               good_l4_checksum1 =
1422                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1423             }
1424
1425           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1426
1427           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1428           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1429
1430           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1431           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1432           error0 = (! good_l4_checksum0
1433                     ? IP6_ERROR_UDP_CHECKSUM + type0
1434                     : error0);
1435           error1 = (! good_l4_checksum1
1436                     ? IP6_ERROR_UDP_CHECKSUM + type1
1437                     : error1);
1438
1439           /* Drop packets from unroutable hosts. */
1440           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1441           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1442               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1443               !ip6_address_is_link_local_unicast(&ip0->src_address))
1444             {
1445               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1446               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1447                         ? IP6_ERROR_SRC_LOOKUP_MISS
1448                         : error0);
1449             }
1450           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1451               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1452               !ip6_address_is_link_local_unicast(&ip1->src_address))
1453             {
1454               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1455               error1 = (ADJ_INDEX_INVALID == src_adj_index1
1456                         ? IP6_ERROR_SRC_LOOKUP_MISS
1457                         : error1);
1458             }
1459
1460           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1461           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1462
1463           p0->error = error_node->errors[error0];
1464           p1->error = error_node->errors[error1];
1465
1466           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1467                                            to_next, n_left_to_next,
1468                                            pi0, pi1, next0, next1);
1469         }
1470
1471       while (n_left_from > 0 && n_left_to_next > 0)
1472         {
1473           vlib_buffer_t * p0;
1474           ip6_header_t * ip0;
1475           udp_header_t * udp0;
1476           u32 pi0, ip_len0, udp_len0, flags0, next0;
1477           i32 len_diff0;
1478           u8 error0, type0, good_l4_checksum0;
1479           u32 udp_offset0;
1480
1481           pi0 = to_next[0] = from[0];
1482           from += 1;
1483           n_left_from -= 1;
1484           to_next += 1;
1485           n_left_to_next -= 1;
1486
1487           p0 = vlib_get_buffer (vm, pi0);
1488
1489           ip0 = vlib_buffer_get_current (p0);
1490
1491           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1492           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1493
1494           flags0 = p0->flags;
1495
1496           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1497           len_diff0 = 0;
1498
1499           /* Skip HBH local processing */
1500           if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1501             {
1502               ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)ip6_next_header(ip0);
1503               next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
1504               type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
1505             }
1506           if (PREDICT_TRUE(IP_PROTOCOL_UDP == ip6_locate_header(p0, ip0,
1507                                                            IP_PROTOCOL_UDP, &udp_offset0)))
1508             {
1509               udp0 = (udp_header_t *)((u8 *)ip0 + udp_offset0);
1510               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1511               good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1512               /* Verify UDP length. */
1513               ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1514               udp_len0 = clib_net_to_host_u16 (udp0->length);
1515               len_diff0 = ip_len0 - udp_len0;
1516             }
1517
1518           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1519           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1520
1521           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1522                              && ! good_l4_checksum0
1523                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1524             {
1525               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1526               good_l4_checksum0 =
1527                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1528             }
1529
1530           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1531
1532           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1533
1534           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1535           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1536           error0 = (! good_l4_checksum0
1537                     ? IP6_ERROR_UDP_CHECKSUM + type0
1538                     : error0);
1539
1540           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1541           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1542               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1543               !ip6_address_is_link_local_unicast(&ip0->src_address))
1544             {
1545               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1546               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1547                         ? IP6_ERROR_SRC_LOOKUP_MISS
1548                         : error0);
1549             }
1550
1551           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1552
1553           p0->error = error_node->errors[error0];
1554
1555           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1556                                            to_next, n_left_to_next,
1557                                            pi0, next0);
1558         }
1559
1560       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1561     }
1562
1563   return frame->n_vectors;
1564 }
1565
1566 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1567   .function = ip6_local,
1568   .name = "ip6-local",
1569   .vector_size = sizeof (u32),
1570
1571   .format_trace = format_ip6_forward_next_trace,
1572
1573   .n_next_nodes = IP_LOCAL_N_NEXT,
1574   .next_nodes = {
1575     [IP_LOCAL_NEXT_DROP] = "error-drop",
1576     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1577     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1578     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1579   },
1580 };
1581
1582 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
1583
1584 void ip6_register_protocol (u32 protocol, u32 node_index)
1585 {
1586   vlib_main_t * vm = vlib_get_main();
1587   ip6_main_t * im = &ip6_main;
1588   ip_lookup_main_t * lm = &im->lookup_main;
1589
1590   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1591   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1592 }
1593
1594 typedef enum {
1595   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1596   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1597   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1598 } ip6_discover_neighbor_next_t;
1599
1600 typedef enum {
1601   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1602   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1603   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
1604 } ip6_discover_neighbor_error_t;
1605
1606 static uword
1607 ip6_discover_neighbor_inline (vlib_main_t * vm,
1608                               vlib_node_runtime_t * node,
1609                               vlib_frame_t * frame,
1610                               int is_glean)
1611 {
1612   vnet_main_t * vnm = vnet_get_main();
1613   ip6_main_t * im = &ip6_main;
1614   ip_lookup_main_t * lm = &im->lookup_main;
1615   u32 * from, * to_next_drop;
1616   uword n_left_from, n_left_to_next_drop;
1617   static f64 time_last_seed_change = -1e100;
1618   static u32 hash_seeds[3];
1619   static uword hash_bitmap[256 / BITS (uword)];
1620   f64 time_now;
1621   int bogus_length;
1622
1623   if (node->flags & VLIB_NODE_FLAG_TRACE)
1624     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1625
1626   time_now = vlib_time_now (vm);
1627   if (time_now - time_last_seed_change > 1e-3)
1628     {
1629       uword i;
1630       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1631                                              sizeof (hash_seeds));
1632       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1633         hash_seeds[i] = r[i];
1634
1635       /* Mark all hash keys as been not-seen before. */
1636       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1637         hash_bitmap[i] = 0;
1638
1639       time_last_seed_change = time_now;
1640     }
1641
1642   from = vlib_frame_vector_args (frame);
1643   n_left_from = frame->n_vectors;
1644
1645   while (n_left_from > 0)
1646     {
1647       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1648                            to_next_drop, n_left_to_next_drop);
1649
1650       while (n_left_from > 0 && n_left_to_next_drop > 0)
1651         {
1652           vlib_buffer_t * p0;
1653           ip6_header_t * ip0;
1654           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1655           uword bm0;
1656           ip_adjacency_t * adj0;
1657           vnet_hw_interface_t * hw_if0;
1658           u32 next0;
1659
1660           pi0 = from[0];
1661
1662           p0 = vlib_get_buffer (vm, pi0);
1663
1664           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1665
1666           ip0 = vlib_buffer_get_current (p0);
1667
1668           adj0 = ip_get_adjacency (lm, adj_index0);
1669
1670           if (!is_glean)
1671             {
1672               ip0->dst_address.as_u64[0] = adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
1673               ip0->dst_address.as_u64[1] = adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
1674             }
1675
1676           a0 = hash_seeds[0];
1677           b0 = hash_seeds[1];
1678           c0 = hash_seeds[2];
1679
1680           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1681           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1682
1683           a0 ^= sw_if_index0;
1684           b0 ^= ip0->dst_address.as_u32[0];
1685           c0 ^= ip0->dst_address.as_u32[1];
1686
1687           hash_v3_mix32 (a0, b0, c0);
1688
1689           b0 ^= ip0->dst_address.as_u32[2];
1690           c0 ^= ip0->dst_address.as_u32[3];
1691
1692           hash_v3_finalize32 (a0, b0, c0);
1693
1694           c0 &= BITS (hash_bitmap) - 1;
1695           c0 = c0 / BITS (uword);
1696           m0 = (uword) 1 << (c0 % BITS (uword));
1697
1698           bm0 = hash_bitmap[c0];
1699           drop0 = (bm0 & m0) != 0;
1700
1701           /* Mark it as seen. */
1702           hash_bitmap[c0] = bm0 | m0;
1703
1704           from += 1;
1705           n_left_from -= 1;
1706           to_next_drop[0] = pi0;
1707           to_next_drop += 1;
1708           n_left_to_next_drop -= 1;
1709
1710           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1711
1712           /* If the interface is link-down, drop the pkt */
1713           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1714             drop0 = 1;
1715
1716           p0->error =
1717             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1718                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1719           if (drop0)
1720             continue;
1721
1722           /*
1723            * the adj has been updated to a rewrite but the node the DPO that got
1724            * us here hasn't - yet. no big deal. we'll drop while we wait.
1725            */
1726           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1727             continue;
1728
1729           {
1730             u32 bi0 = 0;
1731             icmp6_neighbor_solicitation_header_t * h0;
1732             vlib_buffer_t * b0;
1733
1734             h0 = vlib_packet_template_get_packet
1735               (vm, &im->discover_neighbor_packet_template, &bi0);
1736
1737             /*
1738              * Build ethernet header.
1739              * Choose source address based on destination lookup
1740              * adjacency.
1741              */
1742             if (ip6_src_address_for_packet (lm,
1743                                             sw_if_index0,
1744                                             &h0->ip.src_address))
1745               {
1746                 /* There is no address on the interface */
1747                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
1748                 vlib_buffer_free(vm, &bi0, 1);
1749                 continue;
1750               }
1751
1752             /*
1753              * Destination address is a solicited node multicast address.
1754              * We need to fill in
1755              * the low 24 bits with low 24 bits of target's address.
1756              */
1757             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1758             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1759             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1760
1761             h0->neighbor.target_address = ip0->dst_address;
1762
1763             clib_memcpy (h0->link_layer_option.ethernet_address,
1764                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
1765
1766             /* $$$$ appears we need this; why is the checksum non-zero? */
1767             h0->neighbor.icmp.checksum = 0;
1768             h0->neighbor.icmp.checksum =
1769               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1770                                                  &bogus_length);
1771
1772             ASSERT (bogus_length == 0);
1773
1774             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1775             b0 = vlib_get_buffer (vm, bi0);
1776             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1777               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1778
1779             /* Add rewrite/encap string. */
1780             vnet_rewrite_one_header (adj0[0], h0,
1781                                      sizeof (ethernet_header_t));
1782             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1783
1784             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1785
1786             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1787           }
1788         }
1789
1790       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1791                            n_left_to_next_drop);
1792     }
1793
1794   return frame->n_vectors;
1795 }
1796
1797 static uword
1798 ip6_discover_neighbor (vlib_main_t * vm,
1799                        vlib_node_runtime_t * node,
1800                        vlib_frame_t * frame)
1801 {
1802     return (ip6_discover_neighbor_inline(vm, node, frame, 0));
1803 }
1804
1805 static uword
1806 ip6_glean (vlib_main_t * vm,
1807            vlib_node_runtime_t * node,
1808            vlib_frame_t * frame)
1809 {
1810     return (ip6_discover_neighbor_inline(vm, node, frame, 1));
1811 }
1812
1813 static char * ip6_discover_neighbor_error_strings[] = {
1814   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1815   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
1816   = "neighbor solicitations sent",
1817   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
1818     = "no source address for ND solicitation",
1819 };
1820
1821 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
1822   .function = ip6_discover_neighbor,
1823   .name = "ip6-discover-neighbor",
1824   .vector_size = sizeof (u32),
1825
1826   .format_trace = format_ip6_forward_next_trace,
1827
1828   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1829   .error_strings = ip6_discover_neighbor_error_strings,
1830
1831   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1832   .next_nodes = {
1833     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1834     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1835   },
1836 };
1837
1838 VLIB_REGISTER_NODE (ip6_glean_node) = {
1839   .function = ip6_glean,
1840   .name = "ip6-glean",
1841   .vector_size = sizeof (u32),
1842
1843   .format_trace = format_ip6_forward_next_trace,
1844
1845   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1846   .error_strings = ip6_discover_neighbor_error_strings,
1847
1848   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1849   .next_nodes = {
1850     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1851     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1852   },
1853 };
1854
1855 clib_error_t *
1856 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
1857 {
1858   vnet_main_t * vnm = vnet_get_main();
1859   ip6_main_t * im = &ip6_main;
1860   icmp6_neighbor_solicitation_header_t * h;
1861   ip6_address_t * src;
1862   ip_interface_address_t * ia;
1863   ip_adjacency_t * adj;
1864   vnet_hw_interface_t * hi;
1865   vnet_sw_interface_t * si;
1866   vlib_buffer_t * b;
1867   u32 bi = 0;
1868   int bogus_length;
1869
1870   si = vnet_get_sw_interface (vnm, sw_if_index);
1871
1872   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1873     {
1874       return clib_error_return (0, "%U: interface %U down",
1875                                 format_ip6_address, dst,
1876                                 format_vnet_sw_if_index_name, vnm,
1877                                 sw_if_index);
1878     }
1879
1880   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1881   if (! src)
1882     {
1883       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1884       return clib_error_return
1885         (0, "no matching interface address for destination %U (interface %U)",
1886          format_ip6_address, dst,
1887          format_vnet_sw_if_index_name, vnm, sw_if_index);
1888     }
1889
1890   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
1891
1892   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1893
1894   /* Destination address is a solicited node multicast address.  We need to fill in
1895      the low 24 bits with low 24 bits of target's address. */
1896   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1897   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1898   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1899
1900   h->ip.src_address = src[0];
1901   h->neighbor.target_address = dst[0];
1902
1903   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
1904
1905   h->neighbor.icmp.checksum =
1906     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1907   ASSERT(bogus_length == 0);
1908
1909   b = vlib_get_buffer (vm, bi);
1910   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1911
1912   /* Add encapsulation string for software interface (e.g. ethernet header). */
1913   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
1914   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1915   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1916
1917   {
1918     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
1919     u32 * to_next = vlib_frame_vector_args (f);
1920     to_next[0] = bi;
1921     f->n_vectors = 1;
1922     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1923   }
1924
1925   return /* no error */ 0;
1926 }
1927
1928 typedef enum {
1929   IP6_REWRITE_NEXT_DROP,
1930   IP6_REWRITE_NEXT_ICMP_ERROR,
1931 } ip6_rewrite_next_t;
1932
1933 always_inline uword
1934 ip6_rewrite_inline (vlib_main_t * vm,
1935                     vlib_node_runtime_t * node,
1936                     vlib_frame_t * frame,
1937                     int is_midchain)
1938 {
1939   ip_lookup_main_t * lm = &ip6_main.lookup_main;
1940   u32 * from = vlib_frame_vector_args (frame);
1941   u32 n_left_from, n_left_to_next, * to_next, next_index;
1942   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1943
1944   n_left_from = frame->n_vectors;
1945   next_index = node->cached_next_index;
1946   u32 cpu_index = os_get_cpu_number();
1947
1948   while (n_left_from > 0)
1949     {
1950       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1951
1952       while (n_left_from >= 4 && n_left_to_next >= 2)
1953         {
1954           ip_adjacency_t * adj0, * adj1;
1955           vlib_buffer_t * p0, * p1;
1956           ip6_header_t * ip0, * ip1;
1957           u32 pi0, rw_len0, next0, error0, adj_index0;
1958           u32 pi1, rw_len1, next1, error1, adj_index1;
1959           u32 tx_sw_if_index0, tx_sw_if_index1;
1960
1961           /* Prefetch next iteration. */
1962           {
1963             vlib_buffer_t * p2, * p3;
1964
1965             p2 = vlib_get_buffer (vm, from[2]);
1966             p3 = vlib_get_buffer (vm, from[3]);
1967
1968             vlib_prefetch_buffer_header (p2, LOAD);
1969             vlib_prefetch_buffer_header (p3, LOAD);
1970
1971             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1972             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1973
1974             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1975             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1976           }
1977
1978           pi0 = to_next[0] = from[0];
1979           pi1 = to_next[1] = from[1];
1980
1981           from += 2;
1982           n_left_from -= 2;
1983           to_next += 2;
1984           n_left_to_next -= 2;
1985
1986           p0 = vlib_get_buffer (vm, pi0);
1987           p1 = vlib_get_buffer (vm, pi1);
1988
1989           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1990           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1991
1992           /* We should never rewrite a pkt using the MISS adjacency */
1993           ASSERT(adj_index0 && adj_index1);
1994
1995           ip0 = vlib_buffer_get_current (p0);
1996           ip1 = vlib_buffer_get_current (p1);
1997
1998           error0 = error1 = IP6_ERROR_NONE;
1999           next0 = next1 = IP6_REWRITE_NEXT_DROP;
2000
2001           if (PREDICT_TRUE(!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2002             {
2003                 i32 hop_limit0 = ip0->hop_limit;
2004
2005               /* Input node should have reject packets with hop limit 0. */
2006               ASSERT (ip0->hop_limit > 0);
2007
2008               hop_limit0 -= 1;
2009
2010               ip0->hop_limit = hop_limit0;
2011
2012               /*
2013                * If the hop count drops below 1 when forwarding, generate
2014                * an ICMP response.
2015                */
2016               if (PREDICT_FALSE(hop_limit0 <= 0))
2017                 {
2018                   error0 = IP6_ERROR_TIME_EXPIRED;
2019                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2020                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2021                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2022                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2023                 }
2024             }
2025           else
2026             {
2027               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2028             }
2029           if (PREDICT_TRUE(!(p1->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2030           {
2031               i32 hop_limit1 = ip1->hop_limit;
2032
2033               /* Input node should have reject packets with hop limit 0. */
2034               ASSERT (ip1->hop_limit > 0);
2035
2036               hop_limit1 -= 1;
2037
2038               ip1->hop_limit = hop_limit1;
2039
2040               /*
2041                * If the hop count drops below 1 when forwarding, generate
2042                * an ICMP response.
2043                */
2044               if (PREDICT_FALSE(hop_limit1 <= 0))
2045                 {
2046                   error1 = IP6_ERROR_TIME_EXPIRED;
2047                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
2048                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
2049                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
2050                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2051                 }
2052           }
2053           else
2054             {
2055               p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2056             }
2057           adj0 = ip_get_adjacency (lm, adj_index0);
2058           adj1 = ip_get_adjacency (lm, adj_index1);
2059
2060           rw_len0 = adj0[0].rewrite_header.data_bytes;
2061           rw_len1 = adj1[0].rewrite_header.data_bytes;
2062           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2063           vnet_buffer(p1)->ip.save_rewrite_length = rw_len1;
2064
2065           vlib_increment_combined_counter (&adjacency_counters,
2066                                            cpu_index,
2067                                            adj_index0,
2068                                            /* packet increment */ 0,
2069                                            /* byte increment */ rw_len0);
2070           vlib_increment_combined_counter (&adjacency_counters,
2071                                            cpu_index,
2072                                            adj_index1,
2073                                            /* packet increment */ 0,
2074                                            /* byte increment */ rw_len1);
2075
2076           /* Check MTU of outgoing interface. */
2077           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2078                     ? IP6_ERROR_MTU_EXCEEDED
2079                     : error0);
2080           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
2081                     ? IP6_ERROR_MTU_EXCEEDED
2082                     : error1);
2083
2084           /* Don't adjust the buffer for hop count issue; icmp-error node
2085            * wants to see the IP headerr */
2086           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2087             {
2088               p0->current_data -= rw_len0;
2089               p0->current_length += rw_len0;
2090
2091               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2092               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
2093                   tx_sw_if_index0;
2094               next0 = adj0[0].rewrite_header.next_index;
2095
2096               vnet_feature_arc_start(lm->output_feature_arc_index,
2097                                      tx_sw_if_index0, &next0, p0);
2098             }
2099           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
2100             {
2101               p1->current_data -= rw_len1;
2102               p1->current_length += rw_len1;
2103
2104               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2105               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
2106                   tx_sw_if_index1;
2107               next1 = adj1[0].rewrite_header.next_index;
2108
2109               vnet_feature_arc_start(lm->output_feature_arc_index,
2110                                      tx_sw_if_index1, &next1, p1);
2111             }
2112
2113           /* Guess we are only writing on simple Ethernet header. */
2114           vnet_rewrite_two_headers (adj0[0], adj1[0],
2115                                     ip0, ip1,
2116                                     sizeof (ethernet_header_t));
2117
2118           if (is_midchain)
2119           {
2120               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2121               adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
2122           }
2123
2124           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2125                                            to_next, n_left_to_next,
2126                                            pi0, pi1, next0, next1);
2127         }
2128
2129       while (n_left_from > 0 && n_left_to_next > 0)
2130         {
2131           ip_adjacency_t * adj0;
2132           vlib_buffer_t * p0;
2133           ip6_header_t * ip0;
2134           u32 pi0, rw_len0;
2135           u32 adj_index0, next0, error0;
2136           u32 tx_sw_if_index0;
2137
2138           pi0 = to_next[0] = from[0];
2139
2140           p0 = vlib_get_buffer (vm, pi0);
2141
2142           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2143
2144           /* We should never rewrite a pkt using the MISS adjacency */
2145           ASSERT(adj_index0);
2146
2147           adj0 = ip_get_adjacency (lm, adj_index0);
2148
2149           ip0 = vlib_buffer_get_current (p0);
2150
2151           error0 = IP6_ERROR_NONE;
2152           next0 = IP6_REWRITE_NEXT_DROP;
2153
2154           /* Check hop limit */
2155           if (PREDICT_TRUE(!(p0->flags & VNET_BUFFER_LOCALLY_ORIGINATED)))
2156             {
2157               i32 hop_limit0 = ip0->hop_limit;
2158
2159               ASSERT (ip0->hop_limit > 0);
2160
2161               hop_limit0 -= 1;
2162
2163               ip0->hop_limit = hop_limit0;
2164
2165               if (PREDICT_FALSE(hop_limit0 <= 0))
2166                 {
2167                   /*
2168                    * If the hop count drops below 1 when forwarding, generate
2169                    * an ICMP response.
2170                    */
2171                   error0 = IP6_ERROR_TIME_EXPIRED;
2172                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2173                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2174                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2175                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2176                 }
2177             }
2178           else
2179             {
2180               p0->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
2181             }
2182
2183           /* Guess we are only writing on simple Ethernet header. */
2184           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2185
2186           /* Update packet buffer attributes/set output interface. */
2187           rw_len0 = adj0[0].rewrite_header.data_bytes;
2188           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2189
2190           vlib_increment_combined_counter (&adjacency_counters,
2191                                            cpu_index,
2192                                            adj_index0,
2193                                            /* packet increment */ 0,
2194                                            /* byte increment */ rw_len0);
2195
2196           /* Check MTU of outgoing interface. */
2197           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2198                     ? IP6_ERROR_MTU_EXCEEDED
2199                     : error0);
2200
2201           /* Don't adjust the buffer for hop count issue; icmp-error node
2202            * wants to see the IP headerr */
2203           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2204             {
2205               p0->current_data -= rw_len0;
2206               p0->current_length += rw_len0;
2207
2208               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2209
2210               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2211               next0 = adj0[0].rewrite_header.next_index;
2212
2213               vnet_feature_arc_start(lm->output_feature_arc_index,
2214                                      tx_sw_if_index0, &next0, p0);
2215             }
2216
2217           if (is_midchain)
2218           {
2219               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2220           }
2221
2222           p0->error = error_node->errors[error0];
2223
2224           from += 1;
2225           n_left_from -= 1;
2226           to_next += 1;
2227           n_left_to_next -= 1;
2228
2229           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2230                                            to_next, n_left_to_next,
2231                                            pi0, next0);
2232         }
2233
2234       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2235     }
2236
2237   /* Need to do trace after rewrites to pick up new packet data. */
2238   if (node->flags & VLIB_NODE_FLAG_TRACE)
2239     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2240
2241   return frame->n_vectors;
2242 }
2243
2244 static uword
2245 ip6_rewrite (vlib_main_t * vm,
2246              vlib_node_runtime_t * node,
2247              vlib_frame_t * frame)
2248 {
2249   return ip6_rewrite_inline (vm, node, frame,
2250                              /* midchain */ 0);
2251 }
2252
2253 static uword
2254 ip6_midchain (vlib_main_t * vm,
2255               vlib_node_runtime_t * node,
2256               vlib_frame_t * frame)
2257 {
2258   return ip6_rewrite_inline (vm, node, frame,
2259                              /* midchain */ 1);
2260 }
2261
2262 VLIB_REGISTER_NODE (ip6_midchain_node) = {
2263   .function = ip6_midchain,
2264   .name = "ip6-midchain",
2265   .vector_size = sizeof (u32),
2266
2267   .format_trace = format_ip6_forward_next_trace,
2268
2269   .sibling_of = "ip6-rewrite",
2270 };
2271
2272 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain)
2273
2274 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2275   .function = ip6_rewrite,
2276   .name = "ip6-rewrite",
2277   .vector_size = sizeof (u32),
2278
2279   .format_trace = format_ip6_rewrite_trace,
2280
2281   .n_next_nodes = 2,
2282   .next_nodes = {
2283     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2284     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2285   },
2286 };
2287
2288 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite);
2289
2290 /*
2291  * Hop-by-Hop handling
2292  */
2293
2294 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2295
2296 #define foreach_ip6_hop_by_hop_error \
2297 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2298 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2299 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2300
2301 typedef enum {
2302 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2303   foreach_ip6_hop_by_hop_error
2304 #undef _
2305   IP6_HOP_BY_HOP_N_ERROR,
2306 } ip6_hop_by_hop_error_t;
2307
2308 /*
2309  * Primary h-b-h handler trace support
2310  * We work pretty hard on the problem for obvious reasons
2311  */
2312 typedef struct {
2313   u32 next_index;
2314   u32 trace_len;
2315   u8 option_data[256];
2316 } ip6_hop_by_hop_trace_t;
2317
2318 vlib_node_registration_t ip6_hop_by_hop_node;
2319
2320 static char * ip6_hop_by_hop_error_strings[] = {
2321 #define _(sym,string) string,
2322   foreach_ip6_hop_by_hop_error
2323 #undef _
2324 };
2325
2326 static u8 *
2327 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2328 {
2329   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2330   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2331   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2332   ip6_hop_by_hop_header_t *hbh0;
2333   ip6_hop_by_hop_option_t *opt0, *limit0;
2334   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2335
2336   u8 type0;
2337
2338   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2339
2340   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2341               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2342
2343   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2344   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2345
2346   while (opt0 < limit0) {
2347     type0 = opt0->type;
2348     switch (type0) {
2349     case 0: /* Pad, just stop */
2350       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2351       break;
2352
2353     default:
2354       if (hm->trace[type0]) {
2355         s = (*hm->trace[type0])(s, opt0);
2356       } else {
2357         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2358       }
2359       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2360       break;
2361     }
2362   }
2363   return s;
2364 }
2365
2366 always_inline u8 ip6_scan_hbh_options (
2367                                        vlib_buffer_t * b0,
2368                                        ip6_header_t *ip0,
2369                                        ip6_hop_by_hop_header_t *hbh0,
2370                                        ip6_hop_by_hop_option_t *opt0,
2371                                        ip6_hop_by_hop_option_t *limit0,
2372                                        u32 *next0)
2373 {
2374   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2375   u8 type0;
2376   u8 error0 = 0;
2377
2378   while (opt0 < limit0)
2379     {
2380       type0 = opt0->type;
2381       switch (type0)
2382         {
2383         case 0: /* Pad1 */
2384           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2385           continue;
2386         case 1: /* PadN */
2387           break;
2388         default:
2389           if (hm->options[type0])
2390             {
2391               if ((*hm->options[type0])(b0, ip0, opt0) < 0)
2392                 {
2393                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2394                   return(error0);
2395                 }
2396             }
2397           else
2398             {
2399               /* Unrecognized mandatory option, check the two high order bits */
2400               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2401                 {
2402                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2403                   break;
2404                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2405                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2406                   *next0 = IP_LOOKUP_NEXT_DROP;
2407                   break;
2408                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2409                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2410                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2411                   icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2412                                               ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2413                   break;
2414                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2415                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2416                   if (!ip6_address_is_multicast(&ip0->dst_address))
2417                     {
2418                       *next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2419                       icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2420                                                   ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2421                     }
2422                   else
2423                     {
2424                       *next0 =  IP_LOOKUP_NEXT_DROP;
2425                     }
2426                   break;
2427                 }
2428               return(error0);
2429             }
2430         }
2431       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2432     }
2433   return(error0);
2434 }
2435
2436 /*
2437  * Process the Hop-by-Hop Options header
2438  */
2439 static uword
2440 ip6_hop_by_hop (vlib_main_t * vm,
2441                 vlib_node_runtime_t * node,
2442                 vlib_frame_t * frame)
2443 {
2444   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2445   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2446   u32 n_left_from, *from, *to_next;
2447   ip_lookup_next_t next_index;
2448   ip6_main_t * im = &ip6_main;
2449   ip_lookup_main_t *lm = &im->lookup_main;
2450
2451   from = vlib_frame_vector_args (frame);
2452   n_left_from = frame->n_vectors;
2453   next_index = node->cached_next_index;
2454
2455   while (n_left_from > 0) {
2456     u32 n_left_to_next;
2457
2458     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2459
2460     while (n_left_from >= 4 && n_left_to_next >= 2) {
2461       u32 bi0, bi1;
2462       vlib_buffer_t * b0, *b1;
2463       u32 next0, next1;
2464       ip6_header_t * ip0, *ip1;
2465       ip6_hop_by_hop_header_t *hbh0, *hbh1;
2466       ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2467       u8 error0 = 0, error1 = 0;
2468
2469       /* Prefetch next iteration. */
2470       {
2471         vlib_buffer_t * p2, * p3;
2472
2473         p2 = vlib_get_buffer (vm, from[2]);
2474         p3 = vlib_get_buffer (vm, from[3]);
2475
2476         vlib_prefetch_buffer_header (p2, LOAD);
2477         vlib_prefetch_buffer_header (p3, LOAD);
2478
2479         CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2480         CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2481       }
2482
2483       /* Speculatively enqueue b0, b1 to the current next frame */
2484       to_next[0] = bi0 = from[0];
2485       to_next[1] = bi1 = from[1];
2486       from += 2;
2487       to_next += 2;
2488       n_left_from -= 2;
2489       n_left_to_next -= 2;
2490
2491       b0 = vlib_get_buffer (vm, bi0);
2492       b1 = vlib_get_buffer (vm, bi1);
2493
2494       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2495       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2496       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2497       u32 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
2498       ip_adjacency_t *adj1 = ip_get_adjacency(lm, adj_index1);
2499
2500       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2501       next0 = adj0->lookup_next_index;
2502       next1 = adj1->lookup_next_index;
2503
2504       ip0 = vlib_buffer_get_current (b0);
2505       ip1 = vlib_buffer_get_current (b1);
2506       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2507       hbh1 = (ip6_hop_by_hop_header_t *)(ip1+1);
2508       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2509       opt1 = (ip6_hop_by_hop_option_t *)(hbh1+1);
2510       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2511       limit1 = (ip6_hop_by_hop_option_t *)((u8 *)hbh1 + ((hbh1->length + 1) << 3));
2512
2513       /*
2514        * Basic validity checks
2515        */
2516       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2517         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2518         next0 = IP_LOOKUP_NEXT_DROP;
2519         goto outdual;
2520       }
2521       /* Scan the set of h-b-h options, process ones that we understand */
2522       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2523
2524       if ((hbh1->length + 1) << 3 > clib_net_to_host_u16(ip1->payload_length)) {
2525         error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2526         next1 = IP_LOOKUP_NEXT_DROP;
2527         goto outdual;
2528       }
2529       /* Scan the set of h-b-h options, process ones that we understand */
2530       error1 = ip6_scan_hbh_options(b1,ip1,hbh1,opt1,limit1, &next1);
2531
2532     outdual:
2533       /* Has the classifier flagged this buffer for special treatment? */
2534       if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2535         next0 = hm->next_override;
2536
2537       /* Has the classifier flagged this buffer for special treatment? */
2538       if (PREDICT_FALSE((error1 == 0) && (vnet_buffer(b1)->l2_classify.opaque_index & OI_DECAP)))
2539         next1 = hm->next_override;
2540
2541       if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
2542         {
2543           if (b0->flags & VLIB_BUFFER_IS_TRACED) {
2544             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2545             u32 trace_len = (hbh0->length + 1) << 3;
2546             t->next_index = next0;
2547             /* Capture the h-b-h option verbatim */
2548             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2549             t->trace_len = trace_len;
2550             clib_memcpy(t->option_data, hbh0, trace_len);
2551           }
2552           if (b1->flags & VLIB_BUFFER_IS_TRACED) {
2553             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b1, sizeof (*t));
2554             u32 trace_len = (hbh1->length + 1) << 3;
2555             t->next_index = next1;
2556             /* Capture the h-b-h option verbatim */
2557             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2558             t->trace_len = trace_len;
2559             clib_memcpy(t->option_data, hbh1, trace_len);
2560           }
2561
2562         }
2563
2564       b0->error = error_node->errors[error0];
2565       b1->error = error_node->errors[error1];
2566
2567       /* verify speculative enqueue, maybe switch current next frame */
2568       vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0,
2569                                        bi1,next0, next1);
2570     }
2571
2572     while (n_left_from > 0 && n_left_to_next > 0) {
2573       u32 bi0;
2574       vlib_buffer_t * b0;
2575       u32 next0;
2576       ip6_header_t * ip0;
2577       ip6_hop_by_hop_header_t *hbh0;
2578       ip6_hop_by_hop_option_t *opt0, *limit0;
2579       u8 error0 = 0;
2580
2581       /* Speculatively enqueue b0 to the current next frame */
2582       bi0 = from[0];
2583       to_next[0] = bi0;
2584       from += 1;
2585       to_next += 1;
2586       n_left_from -= 1;
2587       n_left_to_next -= 1;
2588
2589       b0 = vlib_get_buffer (vm, bi0);
2590       /*
2591        * Default use the next_index from the adjacency.
2592        * A HBH option rarely redirects to a different node
2593        */
2594       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2595       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2596       next0 = adj0->lookup_next_index;
2597
2598       ip0 = vlib_buffer_get_current (b0);
2599       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2600       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2601       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2602
2603       /*
2604        * Basic validity checks
2605        */
2606       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2607         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2608         next0 = IP_LOOKUP_NEXT_DROP;
2609         goto out0;
2610       }
2611
2612       /* Scan the set of h-b-h options, process ones that we understand */
2613       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2614
2615     out0:
2616       /* Has the classifier flagged this buffer for special treatment? */
2617     if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2618         next0 = hm->next_override;
2619
2620       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2621         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2622         u32 trace_len = (hbh0->length + 1) << 3;
2623         t->next_index = next0;
2624         /* Capture the h-b-h option verbatim */
2625         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2626         t->trace_len = trace_len;
2627         clib_memcpy(t->option_data, hbh0, trace_len);
2628       }
2629
2630       b0->error = error_node->errors[error0];
2631
2632       /* verify speculative enqueue, maybe switch current next frame */
2633       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2634     }
2635     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2636   }
2637   return frame->n_vectors;
2638 }
2639
2640 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2641   .function = ip6_hop_by_hop,
2642   .name = "ip6-hop-by-hop",
2643   .sibling_of = "ip6-lookup",
2644   .vector_size = sizeof (u32),
2645   .format_trace = format_ip6_hop_by_hop_trace,
2646   .type = VLIB_NODE_TYPE_INTERNAL,
2647   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2648   .error_strings = ip6_hop_by_hop_error_strings,
2649   .n_next_nodes = 0,
2650 };
2651
2652 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2653
2654 static clib_error_t *
2655 ip6_hop_by_hop_init (vlib_main_t * vm)
2656 {
2657   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2658   memset(hm->options, 0, sizeof(hm->options));
2659   memset(hm->trace, 0, sizeof(hm->trace));
2660   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2661   return (0);
2662 }
2663
2664 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2665
2666 void ip6_hbh_set_next_override (uword next)
2667 {
2668   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2669
2670   hm->next_override = next;
2671 }
2672
2673 int
2674 ip6_hbh_register_option (u8 option,
2675                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2676                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2677 {
2678   ip6_main_t * im = &ip6_main;
2679   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2680
2681   ASSERT (option < ARRAY_LEN (hm->options));
2682
2683   /* Already registered */
2684   if (hm->options[option])
2685     return (-1);
2686
2687   hm->options[option] = options;
2688   hm->trace[option] = trace;
2689
2690   /* Set global variable */
2691   im->hbh_enabled = 1;
2692
2693   return (0);
2694 }
2695
2696 int
2697 ip6_hbh_unregister_option (u8 option)
2698 {
2699   ip6_main_t * im = &ip6_main;
2700   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2701
2702   ASSERT (option < ARRAY_LEN (hm->options));
2703
2704   /* Not registered */
2705   if (!hm->options[option])
2706     return (-1);
2707
2708   hm->options[option] = NULL;
2709   hm->trace[option] = NULL;
2710
2711   /* Disable global knob if this was the last option configured */
2712   int i;
2713   bool found = false;
2714   for (i = 0; i < 256; i++) {
2715     if (hm->options[option]) {
2716       found = true;
2717       break;
2718     }
2719   }
2720   if (!found)
2721     im->hbh_enabled = 0;
2722
2723   return (0);
2724 }
2725
2726 /* Global IP6 main. */
2727 ip6_main_t ip6_main;
2728
2729 static clib_error_t *
2730 ip6_lookup_init (vlib_main_t * vm)
2731 {
2732   ip6_main_t * im = &ip6_main;
2733   clib_error_t * error;
2734   uword i;
2735
2736   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2737     return error;
2738
2739   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2740     {
2741       u32 j, i0, i1;
2742
2743       i0 = i / 32;
2744       i1 = i % 32;
2745
2746       for (j = 0; j < i0; j++)
2747         im->fib_masks[i].as_u32[j] = ~0;
2748
2749       if (i1)
2750         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2751     }
2752
2753   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2754
2755   if (im->lookup_table_nbuckets == 0)
2756     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2757
2758   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2759
2760   if (im->lookup_table_size == 0)
2761     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2762
2763   BV(clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2764                         "ip6 FIB fwding table",
2765                         im->lookup_table_nbuckets,
2766                         im->lookup_table_size);
2767   BV(clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2768                         "ip6 FIB non-fwding table",
2769                         im->lookup_table_nbuckets,
2770                         im->lookup_table_size);
2771
2772   /* Create FIB with index 0 and table id of 0. */
2773   fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 0);
2774
2775   {
2776     pg_node_t * pn;
2777     pn = pg_get_node (ip6_lookup_node.index);
2778     pn->unformat_edit = unformat_pg_ip6_header;
2779   }
2780
2781   /* Unless explicitly configured, don't process HBH options */
2782   im->hbh_enabled = 0;
2783
2784   {
2785     icmp6_neighbor_solicitation_header_t p;
2786
2787     memset (&p, 0, sizeof (p));
2788
2789     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2790     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2791                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2792     p.ip.protocol = IP_PROTOCOL_ICMP6;
2793     p.ip.hop_limit = 255;
2794     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2795
2796     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2797
2798     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2799     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2800
2801     vlib_packet_template_init (vm,
2802                                &im->discover_neighbor_packet_template,
2803                                &p, sizeof (p),
2804                                /* alloc chunk size */ 8,
2805                                "ip6 neighbor discovery");
2806   }
2807
2808   return error;
2809 }
2810
2811 VLIB_INIT_FUNCTION (ip6_lookup_init);
2812
2813 static clib_error_t *
2814 add_del_ip6_interface_table (vlib_main_t * vm,
2815                              unformat_input_t * input,
2816                              vlib_cli_command_t * cmd)
2817 {
2818   vnet_main_t * vnm = vnet_get_main();
2819   clib_error_t * error = 0;
2820   u32 sw_if_index, table_id;
2821
2822   sw_if_index = ~0;
2823
2824   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2825     {
2826       error = clib_error_return (0, "unknown interface `%U'",
2827                                  format_unformat_error, input);
2828       goto done;
2829     }
2830
2831   if (unformat (input, "%d", &table_id))
2832     ;
2833   else
2834     {
2835       error = clib_error_return (0, "expected table id `%U'",
2836                                  format_unformat_error, input);
2837       goto done;
2838     }
2839
2840   {
2841     u32 fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6,
2842                                                       table_id);
2843
2844     vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
2845     ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
2846   }
2847
2848
2849  done:
2850   return error;
2851 }
2852
2853 /*?
2854  * Place the indicated interface into the supplied IPv6 FIB table (also known
2855  * as a VRF). If the FIB table does not exist, this command creates it. To
2856  * display the current IPv6 FIB table, use the command '<em>show ip6 fib</em>'.
2857  * FIB table will only be displayed if a route has been added to the table, or
2858  * an IP Address is assigned to an interface in the table (which adds a route
2859  * automatically).
2860  *
2861  * @note IP addresses added after setting the interface IP table end up in
2862  * the indicated FIB table. If the IP address is added prior to adding the
2863  * interface to the FIB table, it will NOT be part of the FIB table. Predictable
2864  * but potentially counter-intuitive results occur if you provision interface
2865  * addresses in multiple FIBs. Upon RX, packets will be processed in the last
2866  * IP table ID provisioned. It might be marginally useful to evade source RPF
2867  * drops to put an interface address into multiple FIBs.
2868  *
2869  * @cliexpar
2870  * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id):
2871  * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2}
2872  ?*/
2873 /* *INDENT-OFF* */
2874 VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) = {
2875   .path = "set interface ip6 table",
2876   .function = add_del_ip6_interface_table,
2877   .short_help = "set interface ip6 table <interface> <table-id>"
2878 };
2879 /* *INDENT-ON* */
2880
2881 void
2882 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
2883                                                   u8 *mac)
2884 {
2885   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2886   /* Invert the "u" bit */
2887   ip->as_u8 [8] = mac[0] ^ (1<<1);
2888   ip->as_u8 [9] = mac[1];
2889   ip->as_u8 [10] = mac[2];
2890   ip->as_u8 [11] = 0xFF;
2891   ip->as_u8 [12] = 0xFE;
2892   ip->as_u8 [13] = mac[3];
2893   ip->as_u8 [14] = mac[4];
2894   ip->as_u8 [15] = mac[5];
2895 }
2896
2897 void
2898 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
2899                                                   ip6_address_t *ip)
2900 {
2901   /* Invert the previously inverted "u" bit */
2902   mac[0] = ip->as_u8 [8] ^ (1<<1);
2903   mac[1] = ip->as_u8 [9];
2904   mac[2] = ip->as_u8 [10];
2905   mac[3] = ip->as_u8 [13];
2906   mac[4] = ip->as_u8 [14];
2907   mac[5] = ip->as_u8 [15];
2908 }
2909
2910 static clib_error_t *
2911 test_ip6_link_command_fn (vlib_main_t * vm,
2912                           unformat_input_t * input,
2913                           vlib_cli_command_t * cmd)
2914 {
2915   u8 mac[6];
2916   ip6_address_t _a, *a = &_a;
2917
2918   if (unformat (input, "%U", unformat_ethernet_address, mac))
2919     {
2920       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2921       vlib_cli_output (vm, "Link local address: %U",
2922                        format_ip6_address, a);
2923       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2924       vlib_cli_output (vm, "Original MAC address: %U",
2925                        format_ethernet_address, mac);
2926     }
2927
2928   return 0;
2929 }
2930
2931 /*?
2932  * This command converts the given MAC Address into an IPv6 link-local
2933  * address.
2934  *
2935  * @cliexpar
2936  * Example of how to create an IPv6 link-local address:
2937  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2938  * Link local address: fe80::14d9:e0ff:fe91:7986
2939  * Original MAC address: 16:d9:e0:91:79:86
2940  * @cliexend
2941 ?*/
2942 /* *INDENT-OFF* */
2943 VLIB_CLI_COMMAND (test_link_command, static) = {
2944   .path = "test ip6 link",
2945   .function = test_ip6_link_command_fn,
2946   .short_help = "test ip6 link <mac-address>",
2947 };
2948 /* *INDENT-ON* */
2949
2950 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2951 {
2952   ip6_main_t * im6 = &ip6_main;
2953   ip6_fib_t * fib;
2954   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
2955
2956   if (p == 0)
2957     return -1;
2958
2959   fib = ip6_fib_get (p[0]);
2960
2961   fib->flow_hash_config = flow_hash_config;
2962   return 1;
2963 }
2964
2965 static clib_error_t *
2966 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2967                               unformat_input_t * input,
2968                               vlib_cli_command_t * cmd)
2969 {
2970   int matched = 0;
2971   u32 table_id = 0;
2972   u32 flow_hash_config = 0;
2973   int rv;
2974
2975   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2976     if (unformat (input, "table %d", &table_id))
2977       matched = 1;
2978 #define _(a,v) \
2979     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2980     foreach_flow_hash_bit
2981 #undef _
2982     else break;
2983   }
2984
2985   if (matched == 0)
2986     return clib_error_return (0, "unknown input `%U'",
2987                               format_unformat_error, input);
2988
2989   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2990   switch (rv)
2991     {
2992     case 1:
2993       break;
2994
2995     case -1:
2996       return clib_error_return (0, "no such FIB table %d", table_id);
2997
2998     default:
2999       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3000       break;
3001     }
3002
3003   return 0;
3004 }
3005
3006 /*?
3007  * Configure the set of IPv6 fields used by the flow hash.
3008  *
3009  * @cliexpar
3010  * @parblock
3011  * Example of how to set the flow hash on a given table:
3012  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
3013  *
3014  * Example of display the configured flow hash:
3015  * @cliexstart{show ip6 fib}
3016  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3017  * @::/0
3018  *   unicast-ip6-chain
3019  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
3020  *     [0] [@0]: dpo-drop ip6
3021  * fe80::/10
3022  *   unicast-ip6-chain
3023  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
3024  *     [0] [@2]: dpo-receive
3025  * ff02::1/128
3026  *   unicast-ip6-chain
3027  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
3028  *     [0] [@2]: dpo-receive
3029  * ff02::2/128
3030  *   unicast-ip6-chain
3031  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
3032  *     [0] [@2]: dpo-receive
3033  * ff02::16/128
3034  *   unicast-ip6-chain
3035  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
3036  *     [0] [@2]: dpo-receive
3037  * ff02::1:ff00:0/104
3038  *   unicast-ip6-chain
3039  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
3040  *     [0] [@2]: dpo-receive
3041  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
3042  * @::/0
3043  *   unicast-ip6-chain
3044  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3045  *     [0] [@0]: dpo-drop ip6
3046  * @::a:1:1:0:4/126
3047  *   unicast-ip6-chain
3048  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
3049  *     [0] [@4]: ipv6-glean: af_packet0
3050  * @::a:1:1:0:7/128
3051  *   unicast-ip6-chain
3052  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
3053  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
3054  * fe80::/10
3055  *   unicast-ip6-chain
3056  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
3057  *     [0] [@2]: dpo-receive
3058  * fe80::fe:3eff:fe3e:9222/128
3059  *   unicast-ip6-chain
3060  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
3061  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
3062  * ff02::1/128
3063  *   unicast-ip6-chain
3064  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
3065  *     [0] [@2]: dpo-receive
3066  * ff02::2/128
3067  *   unicast-ip6-chain
3068  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
3069  *     [0] [@2]: dpo-receive
3070  * ff02::16/128
3071  *   unicast-ip6-chain
3072  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
3073  *     [0] [@2]: dpo-receive
3074  * ff02::1:ff00:0/104
3075  *   unicast-ip6-chain
3076  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
3077  *     [0] [@2]: dpo-receive
3078  * @cliexend
3079  * @endparblock
3080 ?*/
3081 /* *INDENT-OFF* */
3082 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
3083     .path = "set ip6 flow-hash",
3084     .short_help =
3085     "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3086     .function = set_ip6_flow_hash_command_fn,
3087 };
3088 /* *INDENT-ON* */
3089
3090 static clib_error_t *
3091 show_ip6_local_command_fn (vlib_main_t * vm,
3092                            unformat_input_t * input,
3093                            vlib_cli_command_t * cmd)
3094 {
3095   ip6_main_t * im = &ip6_main;
3096   ip_lookup_main_t * lm = &im->lookup_main;
3097   int i;
3098
3099   vlib_cli_output (vm, "Protocols handled by ip6_local");
3100   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
3101     {
3102       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3103         vlib_cli_output (vm, "%d", i);
3104     }
3105   return 0;
3106 }
3107
3108
3109
3110 /*?
3111  * Display the set of protocols handled by the local IPv6 stack.
3112  *
3113  * @cliexpar
3114  * Example of how to display local protocol table:
3115  * @cliexstart{show ip6 local}
3116  * Protocols handled by ip6_local
3117  * 17
3118  * 43
3119  * 58
3120  * 115
3121  * @cliexend
3122 ?*/
3123 /* *INDENT-OFF* */
3124 VLIB_CLI_COMMAND (show_ip6_local, static) = {
3125   .path = "show ip6 local",
3126   .function = show_ip6_local_command_fn,
3127   .short_help = "show ip6 local",
3128 };
3129 /* *INDENT-ON* */
3130
3131 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3132                                  u32 table_index)
3133 {
3134   vnet_main_t * vnm = vnet_get_main();
3135   vnet_interface_main_t * im = &vnm->interface_main;
3136   ip6_main_t * ipm = &ip6_main;
3137   ip_lookup_main_t * lm = &ipm->lookup_main;
3138   vnet_classify_main_t * cm = &vnet_classify_main;
3139   ip6_address_t *if_addr;
3140
3141   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3142     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3143
3144   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3145     return VNET_API_ERROR_NO_SUCH_ENTRY;
3146
3147   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3148   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3149
3150   if_addr = ip6_interface_first_address (ipm, sw_if_index, NULL);
3151
3152   if (NULL != if_addr)
3153   {
3154       fib_prefix_t pfx = {
3155           .fp_len = 128,
3156           .fp_proto = FIB_PROTOCOL_IP6,
3157           .fp_addr.ip6 = *if_addr,
3158       };
3159       u32 fib_index;
3160
3161       fib_index = fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
3162                                                       sw_if_index);
3163
3164
3165       if (table_index != (u32) ~0)
3166       {
3167           dpo_id_t dpo = DPO_INVALID;
3168
3169           dpo_set(&dpo,
3170                   DPO_CLASSIFY,
3171                   DPO_PROTO_IP6,
3172                   classify_dpo_create(DPO_PROTO_IP6,
3173                                       table_index));
3174
3175           fib_table_entry_special_dpo_add(fib_index,
3176                                           &pfx,
3177                                           FIB_SOURCE_CLASSIFY,
3178                                           FIB_ENTRY_FLAG_NONE,
3179                                           &dpo);
3180           dpo_reset(&dpo);
3181       }
3182       else
3183       {
3184           fib_table_entry_special_remove(fib_index,
3185                                          &pfx,
3186                                          FIB_SOURCE_CLASSIFY);
3187       }
3188   }
3189
3190   return 0;
3191 }
3192
3193 static clib_error_t *
3194 set_ip6_classify_command_fn (vlib_main_t * vm,
3195                              unformat_input_t * input,
3196                              vlib_cli_command_t * cmd)
3197 {
3198   u32 table_index = ~0;
3199   int table_index_set = 0;
3200   u32 sw_if_index = ~0;
3201   int rv;
3202
3203   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3204     if (unformat (input, "table-index %d", &table_index))
3205       table_index_set = 1;
3206     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3207                        vnet_get_main(), &sw_if_index))
3208         ;
3209     else
3210         break;
3211   }
3212
3213   if (table_index_set == 0)
3214       return clib_error_return (0, "classify table-index must be specified");
3215
3216   if (sw_if_index == ~0)
3217     return clib_error_return (0, "interface / subif must be specified");
3218
3219   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3220
3221   switch (rv)
3222     {
3223     case 0:
3224       break;
3225
3226     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3227       return clib_error_return (0, "No such interface");
3228
3229     case VNET_API_ERROR_NO_SUCH_ENTRY:
3230       return clib_error_return (0, "No such classifier table");
3231     }
3232   return 0;
3233 }
3234
3235 /*?
3236  * Assign a classification table to an interface. The classification
3237  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3238  * commands. Once the table is create, use this command to filter packets
3239  * on an interface.
3240  *
3241  * @cliexpar
3242  * Example of how to assign a classification table to an interface:
3243  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3244 ?*/
3245 /* *INDENT-OFF* */
3246 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3247     .path = "set ip6 classify",
3248     .short_help =
3249     "set ip6 classify intfc <interface> table-index <classify-idx>",
3250     .function = set_ip6_classify_command_fn,
3251 };
3252 /* *INDENT-ON* */
3253
3254 static clib_error_t *
3255 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3256 {
3257   ip6_main_t * im = &ip6_main;
3258   uword heapsize = 0;
3259   u32 tmp;
3260   u32 nbuckets = 0;
3261
3262   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3263     if (unformat (input, "hash-buckets %d", &tmp))
3264       nbuckets = tmp;
3265     else if (unformat (input, "heap-size %dm", &tmp))
3266       heapsize = ((u64)tmp) << 20;
3267     else if (unformat (input, "heap-size %dM", &tmp))
3268       heapsize = ((u64)tmp) << 20;
3269     else if (unformat (input, "heap-size %dg", &tmp))
3270       heapsize = ((u64)tmp) << 30;
3271     else if (unformat (input, "heap-size %dG", &tmp))
3272       heapsize = ((u64)tmp) << 30;
3273     else
3274       return clib_error_return (0, "unknown input '%U'",
3275                                 format_unformat_error, input);
3276   }
3277
3278   im->lookup_table_nbuckets = nbuckets;
3279   im->lookup_table_size = heapsize;
3280
3281   return 0;
3282 }
3283
3284 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");