vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45 #include <vnet/fib/fib_table.h>
  46 #include <vnet/fib/ip6_fib.h>
  47 #include <vnet/dpo/load_balance.h>
  48 #include <vnet/dpo/classify_dpo.h>
  49
  50 #include <vppinfra/bihash_template.c>
  51
  52 /**
  53  * @file
  54  * @brief IPv6 Forwarding.
  55  *
  56  * This file contains the source code for IPv6 forwarding.
  57  */
  58
  59 void
  60 ip6_forward_next_trace (vlib_main_t * vm,
  61                         vlib_node_runtime_t * node,
  62                         vlib_frame_t * frame,
  63                         vlib_rx_or_tx_t which_adj_index);
  64
  65 always_inline uword
  66 ip6_lookup_inline (vlib_main_t * vm,
  67                    vlib_node_runtime_t * node,
  68                    vlib_frame_t * frame)
  69 {
  70   ip6_main_t * im = &ip6_main;
  71   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  72   u32 n_left_from, n_left_to_next, * from, * to_next;
  73   ip_lookup_next_t next;
  74   u32 cpu_index = os_get_cpu_number();
  75
  76   from = vlib_frame_vector_args (frame);
  77   n_left_from = frame->n_vectors;
  78   next = node->cached_next_index;
  79
  80   while (n_left_from > 0)
  81     {
  82       vlib_get_next_frame (vm, node, next,
  83                            to_next, n_left_to_next);
  84
  85       while (n_left_from >= 4 && n_left_to_next >= 2)
  86         {
  87           vlib_buffer_t * p0, * p1;
  88           u32 pi0, pi1, lbi0, lbi1, wrong_next;
  89           ip_lookup_next_t next0, next1;
  90           ip6_header_t * ip0, * ip1;
  91           ip6_address_t * dst_addr0, * dst_addr1;
  92           u32 fib_index0, fib_index1;
  93           u32 flow_hash_config0, flow_hash_config1;
  94           const dpo_id_t *dpo0, *dpo1;
  95           const load_balance_t *lb0, *lb1;
  96
  97           /* Prefetch next iteration. */
  98           {
  99             vlib_buffer_t * p2, * p3;
 100
 101             p2 = vlib_get_buffer (vm, from[2]);
 102             p3 = vlib_get_buffer (vm, from[3]);
 103
 104             vlib_prefetch_buffer_header (p2, LOAD);
 105             vlib_prefetch_buffer_header (p3, LOAD);
 106             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 107             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 108           }
 109
 110           pi0 = to_next[0] = from[0];
 111           pi1 = to_next[1] = from[1];
 112
 113           p0 = vlib_get_buffer (vm, pi0);
 114           p1 = vlib_get_buffer (vm, pi1);
 115
 116           ip0 = vlib_buffer_get_current (p0);
 117           ip1 = vlib_buffer_get_current (p1);
 118
 119           dst_addr0 = &ip0->dst_address;
 120           dst_addr1 = &ip1->dst_address;
 121
 122           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 123           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 124
 125           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 126             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 127           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 128             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 129
 130           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 131           lbi1 = ip6_fib_table_fwding_lookup (im, fib_index1, dst_addr1);
 132
 133           lb0 = load_balance_get (lbi0);
 134           lb1 = load_balance_get (lbi1);
 135
 136           vnet_buffer (p0)->ip.flow_hash =
 137             vnet_buffer(p1)->ip.flow_hash = 0;
 138
 139           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 140             {
 141               flow_hash_config0 = lb0->lb_hash_config;
 142               vnet_buffer (p0)->ip.flow_hash =
 143                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 144             }
 145           if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
 146             {
 147               flow_hash_config1 = lb1->lb_hash_config;
 148               vnet_buffer (p1)->ip.flow_hash =
 149                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 150             }
 151
 152           ASSERT (lb0->lb_n_buckets > 0);
 153           ASSERT (lb1->lb_n_buckets > 0);
 154           ASSERT (is_pow2 (lb0->lb_n_buckets));
 155           ASSERT (is_pow2 (lb1->lb_n_buckets));
 156           dpo0 = load_balance_get_bucket_i(lb0,
 157                                            (vnet_buffer (p0)->ip.flow_hash &
 158                                             lb0->lb_n_buckets_minus_1));
 159           dpo1 = load_balance_get_bucket_i(lb1,
 160                                            (vnet_buffer (p1)->ip.flow_hash &
 161                                             lb1->lb_n_buckets_minus_1));
 162
 163           next0 = dpo0->dpoi_next_node;
 164           next1 = dpo1->dpoi_next_node;
 165
 166           /* Only process the HBH Option Header if explicitly configured to do so */
 167           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 168             {
 169               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 170                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 171             }
 172           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 173             {
 174               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 175                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 176             }
 177           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 178           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 179
 180           vlib_increment_combined_counter
 181               (cm, cpu_index, lbi0, 1,
 182                vlib_buffer_length_in_chain (vm, p0));
 183           vlib_increment_combined_counter
 184               (cm, cpu_index, lbi1, 1,
 185                vlib_buffer_length_in_chain (vm, p1));
 186
 187           from += 2;
 188           to_next += 2;
 189           n_left_to_next -= 2;
 190           n_left_from -= 2;
 191
 192           wrong_next = (next0 != next) + 2*(next1 != next);
 193           if (PREDICT_FALSE (wrong_next != 0))
 194             {
 195               switch (wrong_next)
 196                 {
 197                 case 1:
 198                   /* A B A */
 199                   to_next[-2] = pi1;
 200                   to_next -= 1;
 201                   n_left_to_next += 1;
 202                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 203                   break;
 204
 205                 case 2:
 206                   /* A A B */
 207                   to_next -= 1;
 208                   n_left_to_next += 1;
 209                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 210                   break;
 211
 212                 case 3:
 213                   /* A B C */
 214                   to_next -= 2;
 215                   n_left_to_next += 2;
 216                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 217                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 218                   if (next0 == next1)
 219                     {
 220                       /* A B B */
 221                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 222                       next = next1;
 223                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 224                     }
 225                 }
 226             }
 227         }
 228
 229       while (n_left_from > 0 && n_left_to_next > 0)
 230         {
 231           vlib_buffer_t * p0;
 232           ip6_header_t * ip0;
 233           u32 pi0, lbi0;
 234           ip_lookup_next_t next0;
 235           load_balance_t * lb0;
 236           ip6_address_t * dst_addr0;
 237           u32 fib_index0, flow_hash_config0;
 238           const dpo_id_t *dpo0;
 239
 240           pi0 = from[0];
 241           to_next[0] = pi0;
 242
 243           p0 = vlib_get_buffer (vm, pi0);
 244
 245           ip0 = vlib_buffer_get_current (p0);
 246
 247           dst_addr0 = &ip0->dst_address;
 248
 249           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 250           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 251             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 252
 253           flow_hash_config0 =
 254               ip6_fib_get (fib_index0)->flow_hash_config;
 255
 256           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 257
 258           lb0 = load_balance_get (lbi0);
 259
 260           vnet_buffer (p0)->ip.flow_hash = 0;
 261
 262           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 263             {
 264               flow_hash_config0 = lb0->lb_hash_config;
 265               vnet_buffer (p0)->ip.flow_hash =
 266                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 267             }
 268
 269           ASSERT (lb0->lb_n_buckets > 0);
 270           ASSERT (is_pow2 (lb0->lb_n_buckets));
 271           dpo0 = load_balance_get_bucket_i(lb0,
 272                                            (vnet_buffer (p0)->ip.flow_hash &
 273                                             lb0->lb_n_buckets_minus_1));
 274           next0 = dpo0->dpoi_next_node;
 275
 276           /* Only process the HBH Option Header if explicitly configured to do so */
 277           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 278             {
 279               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 280                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 281             }
 282           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 283
 284           vlib_increment_combined_counter
 285               (cm, cpu_index, lbi0, 1,
 286                vlib_buffer_length_in_chain (vm, p0));
 287
 288           from += 1;
 289           to_next += 1;
 290           n_left_to_next -= 1;
 291           n_left_from -= 1;
 292
 293           if (PREDICT_FALSE (next0 != next))
 294             {
 295               n_left_to_next += 1;
 296               vlib_put_next_frame (vm, node, next, n_left_to_next);
 297               next = next0;
 298               vlib_get_next_frame (vm, node, next,
 299                                    to_next, n_left_to_next);
 300               to_next[0] = pi0;
 301               to_next += 1;
 302               n_left_to_next -= 1;
 303             }
 304         }
 305
 306       vlib_put_next_frame (vm, node, next, n_left_to_next);
 307     }
 308
 309   if (node->flags & VLIB_NODE_FLAG_TRACE)
 310     ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 311
 312   return frame->n_vectors;
 313 }
 314
 315 static void
 316 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 317                           ip6_main_t * im, u32 fib_index,
 318                           ip_interface_address_t * a)
 319 {
 320   ip_lookup_main_t * lm = &im->lookup_main;
 321   ip6_address_t * address = ip_interface_address_get_address (lm, a);
 322   fib_prefix_t pfx = {
 323       .fp_len = a->address_length,
 324       .fp_proto = FIB_PROTOCOL_IP6,
 325       .fp_addr.ip6 = *address,
 326   };
 327
 328   a->neighbor_probe_adj_index = ~0;
 329   if (a->address_length < 128)
 330   {
 331       fib_node_index_t fei;
 332
 333       fei = fib_table_entry_update_one_path(fib_index,
 334                                             &pfx,
 335                                             FIB_SOURCE_INTERFACE,
 336                                             (FIB_ENTRY_FLAG_CONNECTED |
 337                                              FIB_ENTRY_FLAG_ATTACHED),
 338                                             FIB_PROTOCOL_IP6,
 339                                             NULL, /* No next-hop address */
 340                                             sw_if_index,
 341                                             ~0, // invalid FIB index
 342                                             1,
 343                                             NULL, // no label stack
 344                                             FIB_ROUTE_PATH_FLAG_NONE);
 345       a->neighbor_probe_adj_index = fib_entry_get_adj(fei);
 346   }
 347
 348   pfx.fp_len = 128;
 349   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 350   {
 351       u32 classify_table_index =
 352           lm->classify_table_index_by_sw_if_index [sw_if_index];
 353       if (classify_table_index != (u32) ~0)
 354       {
 355           dpo_id_t dpo = DPO_INVALID;
 356
 357           dpo_set(&dpo,
 358                   DPO_CLASSIFY,
 359                   DPO_PROTO_IP6,
 360                   classify_dpo_create(DPO_PROTO_IP6, classify_table_index));
 361
 362           fib_table_entry_special_dpo_add(fib_index,
 363                                           &pfx,
 364                                           FIB_SOURCE_CLASSIFY,
 365                                           FIB_ENTRY_FLAG_NONE,
 366                                           &dpo);
 367           dpo_reset(&dpo);
 368       }
 369   }
 370
 371   fib_table_entry_update_one_path(fib_index,
 372                                   &pfx,
 373                                   FIB_SOURCE_INTERFACE,
 374                                   (FIB_ENTRY_FLAG_CONNECTED |
 375                                    FIB_ENTRY_FLAG_LOCAL),
 376                                   FIB_PROTOCOL_IP6,
 377                                   &pfx.fp_addr,
 378                                   sw_if_index,
 379                                   ~0, // invalid FIB index
 380                                   1,
 381                                   NULL,
 382                                   FIB_ROUTE_PATH_FLAG_NONE);
 383 }
 384
 385 static void
 386 ip6_del_interface_routes (ip6_main_t * im,
 387                           u32 fib_index,
 388                           ip6_address_t * address,
 389                           u32 address_length)
 390 {
 391     fib_prefix_t pfx = {
 392         .fp_len = address_length,
 393         .fp_proto = FIB_PROTOCOL_IP6,
 394         .fp_addr.ip6 = *address,
 395     };
 396
 397     if (pfx.fp_len < 128)
 398     {
 399         fib_table_entry_delete(fib_index,
 400                                &pfx,
 401                                FIB_SOURCE_INTERFACE);
 402
 403     }
 404
 405     pfx.fp_len = 128;
 406     fib_table_entry_delete(fib_index,
 407                            &pfx,
 408                            FIB_SOURCE_INTERFACE);
 409 }
 410
 411 void
 412 ip6_sw_interface_enable_disable (u32 sw_if_index,
 413                                  u32 is_enable)
 414 {
 415   ip6_main_t * im = &ip6_main;
 416
 417   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 418
 419   /*
 420    * enable/disable only on the 1<->0 transition
 421    */
 422   if (is_enable)
 423     {
 424       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 425         return;
 426     }
 427   else
 428     {
 429       ASSERT(im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 430       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 431         return;
 432     }
 433
 434   vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index,
 435                                is_enable, 0, 0);
 436
 437   vnet_feature_enable_disable ("ip6-multicast", "ip6-lookup", sw_if_index,
 438                                is_enable, 0, 0);
 439
 440 }
 441
 442 /* get first interface address */
 443 ip6_address_t *
 444 ip6_interface_first_address (ip6_main_t * im,
 445                              u32 sw_if_index,
 446                              ip_interface_address_t ** result_ia)
 447 {
 448   ip_lookup_main_t * lm = &im->lookup_main;
 449   ip_interface_address_t * ia = 0;
 450   ip6_address_t * result = 0;
 451
 452   foreach_ip_interface_address (lm, ia, sw_if_index,
 453                                 1 /* honor unnumbered */,
 454   ({
 455     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 456     result = a;
 457     break;
 458   }));
 459   if (result_ia)
 460     *result_ia = result ? ia : 0;
 461   return result;
 462 }
 463
 464 clib_error_t *
 465 ip6_add_del_interface_address (vlib_main_t * vm,
 466                                u32 sw_if_index,
 467                                ip6_address_t * address,
 468                                u32 address_length,
 469                                u32 is_del)
 470 {
 471   vnet_main_t * vnm = vnet_get_main();
 472   ip6_main_t * im = &ip6_main;
 473   ip_lookup_main_t * lm = &im->lookup_main;
 474   clib_error_t * error;
 475   u32 if_address_index;
 476   ip6_address_fib_t ip6_af, * addr_fib = 0;
 477
 478   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 479   ip6_addr_fib_init (&ip6_af, address,
 480                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 481   vec_add1 (addr_fib, ip6_af);
 482
 483   {
 484     uword elts_before = pool_elts (lm->if_address_pool);
 485
 486     error = ip_interface_address_add_del
 487       (lm,
 488        sw_if_index,
 489        addr_fib,
 490        address_length,
 491        is_del,
 492        &if_address_index);
 493     if (error)
 494       goto done;
 495
 496     /* Pool did not grow: add duplicate address. */
 497     if (elts_before == pool_elts (lm->if_address_pool))
 498       goto done;
 499   }
 500
 501   ip6_sw_interface_enable_disable(sw_if_index, !is_del);
 502
 503   if (is_del)
 504       ip6_del_interface_routes (im, ip6_af.fib_index, address,
 505                                 address_length);
 506   else
 507       ip6_add_interface_routes (vnm, sw_if_index,
 508                                 im, ip6_af.fib_index,
 509                                 pool_elt_at_index (lm->if_address_pool, if_address_index));
 510
 511   {
 512     ip6_add_del_interface_address_callback_t * cb;
 513     vec_foreach (cb, im->add_del_interface_address_callbacks)
 514       cb->function (im, cb->function_opaque, sw_if_index,
 515                     address, address_length,
 516                     if_address_index,
 517                     is_del);
 518   }
 519
 520  done:
 521   vec_free (addr_fib);
 522   return error;
 523 }
 524
 525 clib_error_t *
 526 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
 527                                 u32 sw_if_index,
 528                                 u32 flags)
 529 {
 530   ip6_main_t * im = &ip6_main;
 531   ip_interface_address_t * ia;
 532   ip6_address_t * a;
 533   u32 is_admin_up, fib_index;
 534
 535   /* Fill in lookup tables with default table (0). */
 536   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 537
 538   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
 539
 540   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 541
 542   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 543
 544   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 545                                 0 /* honor unnumbered */,
 546   ({
 547     a = ip_interface_address_get_address (&im->lookup_main, ia);
 548     if (is_admin_up)
 549       ip6_add_interface_routes (vnm, sw_if_index,
 550                                 im, fib_index,
 551                                 ia);
 552     else
 553       ip6_del_interface_routes (im, fib_index,
 554                                 a, ia->address_length);
 555   }));
 556
 557   return 0;
 558 }
 559
 560 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 561
 562 /* Built-in ip6 unicast rx feature path definition */
 563 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 564 {
 565   .arc_name  = "ip6-unicast",
 566   .start_nodes = VNET_FEATURES ("ip6-input"),
 567   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 568 };
 569
 570 VNET_FEATURE_INIT (ip6_flow_classify, static) = {
 571   .arc_name = "ip6-unicast",
 572   .node_name = "ip6-flow-classify",
 573   .runs_before = VNET_FEATURES ("ip6-inacl"),
 574 };
 575
 576 VNET_FEATURE_INIT (ip6_inacl, static) = {
 577   .arc_name = "ip6-unicast",
 578   .node_name = "ip6-inacl",
 579   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 580 };
 581
 582 VNET_FEATURE_INIT (ip6_policer_classify, static) = {
 583   .arc_name = "ip6-unicast",
 584   .node_name = "ip6-policer-classify",
 585   .runs_before = VNET_FEATURES ("ipsec-input-ip6"),
 586 };
 587
 588 VNET_FEATURE_INIT (ip6_ipsec, static) = {
 589   .arc_name = "ip6-unicast",
 590   .node_name = "ipsec-input-ip6",
 591   .runs_before = VNET_FEATURES ("l2tp-decap"),
 592 };
 593
 594 VNET_FEATURE_INIT (ip6_l2tp, static) = {
 595   .arc_name = "ip6-unicast",
 596   .node_name = "l2tp-decap",
 597   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 598 };
 599
 600 VNET_FEATURE_INIT (ip6_vpath, static) = {
 601   .arc_name = "ip6-unicast",
 602   .node_name = "vpath-input-ip6",
 603   .runs_before = VNET_FEATURES ("ip6-lookup"),
 604 };
 605
 606 VNET_FEATURE_INIT (ip6_lookup, static) = {
 607   .arc_name = "ip6-unicast",
 608   .node_name = "ip6-lookup",
 609   .runs_before = VNET_FEATURES ("ip6-drop"),
 610 };
 611
 612 VNET_FEATURE_INIT (ip6_drop, static) = {
 613   .arc_name = "ip6-unicast",
 614   .node_name = "ip6-drop",
 615   .runs_before = 0,  /*last feature*/
 616 };
 617
 618 /* Built-in ip6 multicast rx feature path definition (none now) */
 619 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 620 {
 621   .arc_name  = "ip6-multicast",
 622   .start_nodes = VNET_FEATURES ("ip6-input"),
 623   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 624 };
 625
 626 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 627   .arc_name = "ip6-multicast",
 628   .node_name = "vpath-input-ip6",
 629   .runs_before = VNET_FEATURES ("ip6-lookup"),
 630 };
 631
 632 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 633   .arc_name = "ip6-multicast",
 634   .node_name = "ip6-lookup",
 635   .runs_before = VNET_FEATURES ("ip6-drop"),
 636 };
 637
 638 VNET_FEATURE_INIT (ip6_drop_mc, static) = {
 639   .arc_name = "ip6-multicast",
 640   .node_name = "ip6-drop",
 641   .runs_before = 0, /* last feature */
 642 };
 643
 644 /* Built-in ip4 tx feature path definition */
 645 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 646 {
 647   .arc_name  = "ip6-output",
 648   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain"),
 649   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 650 };
 651
 652 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 653   .arc_name = "ip6-output",
 654   .node_name = "ipsec-output-ip6",
 655   .runs_before = VNET_FEATURES ("interface-output"),
 656 };
 657
 658 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 659   .arc_name = "ip6-output",
 660   .node_name = "interface-output",
 661   .runs_before = 0, /* not before any other features */
 662 };
 663
 664 clib_error_t *
 665 ip6_sw_interface_add_del (vnet_main_t * vnm,
 666                           u32 sw_if_index,
 667                           u32 is_add)
 668 {
 669   vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index,
 670                                is_add, 0, 0);
 671
 672   vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index,
 673                                is_add, 0, 0);
 674
 675   vnet_feature_enable_disable ("ip6-output", "interface-output", sw_if_index,
 676                                is_add, 0, 0);
 677
 678   return /* no error */ 0;
 679 }
 680
 681 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 682
 683 static uword
 684 ip6_lookup (vlib_main_t * vm,
 685             vlib_node_runtime_t * node,
 686             vlib_frame_t * frame)
 687 {
 688   return ip6_lookup_inline (vm, node, frame);
 689 }
 690
 691 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
 692
 693 VLIB_REGISTER_NODE (ip6_lookup_node) = {
 694   .function = ip6_lookup,
 695   .name = "ip6-lookup",
 696   .vector_size = sizeof (u32),
 697
 698   .format_trace = format_ip6_lookup_trace,
 699
 700   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 701   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 702 };
 703
 704 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
 705
 706 always_inline uword
 707 ip6_load_balance (vlib_main_t * vm,
 708                   vlib_node_runtime_t * node,
 709                   vlib_frame_t * frame)
 710 {
 711   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
 712   u32 n_left_from, n_left_to_next, * from, * to_next;
 713   ip_lookup_next_t next;
 714   u32 cpu_index = os_get_cpu_number();
 715   ip6_main_t * im = &ip6_main;
 716
 717   from = vlib_frame_vector_args (frame);
 718   n_left_from = frame->n_vectors;
 719   next = node->cached_next_index;
 720
 721   if (node->flags & VLIB_NODE_FLAG_TRACE)
 722       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 723
 724   while (n_left_from > 0)
 725     {
 726       vlib_get_next_frame (vm, node, next,
 727                            to_next, n_left_to_next);
 728
 729
 730       while (n_left_from >= 4 && n_left_to_next >= 2)
 731         {
 732           ip_lookup_next_t next0, next1;
 733           const load_balance_t *lb0, *lb1;
 734           vlib_buffer_t * p0, *p1;
 735           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 736           const ip6_header_t *ip0, *ip1;
 737           const dpo_id_t *dpo0, *dpo1;
 738
 739           /* Prefetch next iteration. */
 740           {
 741             vlib_buffer_t * p2, * p3;
 742
 743             p2 = vlib_get_buffer (vm, from[2]);
 744             p3 = vlib_get_buffer (vm, from[3]);
 745
 746             vlib_prefetch_buffer_header (p2, STORE);
 747             vlib_prefetch_buffer_header (p3, STORE);
 748
 749             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 750             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 751           }
 752
 753           pi0 = to_next[0] = from[0];
 754           pi1 = to_next[1] = from[1];
 755
 756           from += 2;
 757           n_left_from -= 2;
 758           to_next += 2;
 759           n_left_to_next -= 2;
 760
 761           p0 = vlib_get_buffer (vm, pi0);
 762           p1 = vlib_get_buffer (vm, pi1);
 763
 764           ip0 = vlib_buffer_get_current (p0);
 765           ip1 = vlib_buffer_get_current (p1);
 766           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 767           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 768
 769           lb0 = load_balance_get(lbi0);
 770           lb1 = load_balance_get(lbi1);
 771
 772           /*
 773            * this node is for via FIBs we can re-use the hash value from the
 774            * to node if present.
 775            * We don't want to use the same hash value at each level in the recursion
 776            * graph as that would lead to polarisation
 777            */
 778           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 779           hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
 780
 781           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 782           {
 783               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 784               {
 785                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 786               }
 787               else
 788               {
 789                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 790               }
 791           }
 792           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 793           {
 794               if (PREDICT_TRUE (vnet_buffer(p1)->ip.flow_hash))
 795               {
 796                   hc1 = vnet_buffer(p1)->ip.flow_hash = vnet_buffer(p1)->ip.flow_hash >> 1;
 797               }
 798               else
 799               {
 800                   hc1 = vnet_buffer(p1)->ip.flow_hash = ip6_compute_flow_hash(ip1, hc1);
 801               }
 802           }
 803
 804           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 805           dpo1 = load_balance_get_bucket_i(lb1, hc1 & (lb1->lb_n_buckets_minus_1));
 806
 807           next0 = dpo0->dpoi_next_node;
 808           next1 = dpo1->dpoi_next_node;
 809
 810           /* Only process the HBH Option Header if explicitly configured to do so */
 811           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 812             {
 813               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 814                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 815             }
 816           /* Only process the HBH Option Header if explicitly configured to do so */
 817           if (PREDICT_FALSE(ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 818             {
 819               next1 = (dpo_is_adj(dpo1) && im->hbh_enabled) ?
 820                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 821             }
 822
 823           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 824           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 825
 826           vlib_increment_combined_counter
 827               (cm, cpu_index, lbi0, 1,
 828                vlib_buffer_length_in_chain (vm, p0));
 829           vlib_increment_combined_counter
 830               (cm, cpu_index, lbi1, 1,
 831                vlib_buffer_length_in_chain (vm, p1));
 832
 833           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 834                                            to_next, n_left_to_next,
 835                                            pi0, pi1, next0, next1);
 836        }
 837
 838       while (n_left_from > 0 && n_left_to_next > 0)
 839         {
 840           ip_lookup_next_t next0;
 841           const load_balance_t *lb0;
 842           vlib_buffer_t * p0;
 843           u32 pi0, lbi0, hc0;
 844           const ip6_header_t *ip0;
 845           const dpo_id_t *dpo0;
 846
 847           pi0 = from[0];
 848           to_next[0] = pi0;
 849           from += 1;
 850           to_next += 1;
 851           n_left_to_next -= 1;
 852           n_left_from -= 1;
 853
 854           p0 = vlib_get_buffer (vm, pi0);
 855
 856           ip0 = vlib_buffer_get_current (p0);
 857           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 858
 859           lb0 = load_balance_get(lbi0);
 860
 861           hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
 862           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 863           {
 864               if (PREDICT_TRUE (vnet_buffer(p0)->ip.flow_hash))
 865               {
 866                   hc0 = vnet_buffer(p0)->ip.flow_hash = vnet_buffer(p0)->ip.flow_hash >> 1;
 867               }
 868               else
 869               {
 870                   hc0 = vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 871               }
 872           }
 873           dpo0 = load_balance_get_bucket_i(lb0, hc0 & (lb0->lb_n_buckets_minus_1));
 874
 875           next0 = dpo0->dpoi_next_node;
 876           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 877
 878           /* Only process the HBH Option Header if explicitly configured to do so */
 879           if (PREDICT_FALSE(ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 880             {
 881               next0 = (dpo_is_adj(dpo0) && im->hbh_enabled) ?
 882                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 883             }
 884
 885           vlib_increment_combined_counter
 886               (cm, cpu_index, lbi0, 1,
 887                vlib_buffer_length_in_chain (vm, p0));
 888
 889           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 890                                            to_next, n_left_to_next,
 891                                            pi0, next0);
 892         }
 893
 894       vlib_put_next_frame (vm, node, next, n_left_to_next);
 895     }
 896
 897   return frame->n_vectors;
 898 }
 899
 900 VLIB_REGISTER_NODE (ip6_load_balance_node) = {
 901   .function = ip6_load_balance,
 902   .name = "ip6-load-balance",
 903   .vector_size = sizeof (u32),
 904   .sibling_of = "ip6-lookup",
 905
 906   .format_trace = format_ip6_lookup_trace,
 907 };
 908
 909 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance)
 910
 911 typedef struct {
 912   /* Adjacency taken. */
 913   u32 adj_index;
 914   u32 flow_hash;
 915   u32 fib_index;
 916
 917   /* Packet data, possibly *after* rewrite. */
 918   u8 packet_data[128 - 1*sizeof(u32)];
 919 } ip6_forward_next_trace_t;
 920
 921 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
 922 {
 923   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 924   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 925   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 926   uword indent = format_get_indent (s);
 927
 928   s = format(s, "%U%U",
 929              format_white_space, indent,
 930              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 931   return s;
 932 }
 933
 934 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
 935 {
 936   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 937   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 938   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 939   uword indent = format_get_indent (s);
 940
 941   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 942               t->fib_index, t->adj_index, t->flow_hash);
 943   s = format(s, "\n%U%U",
 944              format_white_space, indent,
 945              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 946   return s;
 947 }
 948
 949
 950 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
 951 {
 952   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 953   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 954   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 955   vnet_main_t * vnm = vnet_get_main();
 956   uword indent = format_get_indent (s);
 957
 958   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 959               t->fib_index, t->adj_index, format_ip_adjacency,
 960               t->adj_index, FORMAT_IP_ADJACENCY_NONE,
 961               t->flow_hash);
 962   s = format (s, "\n%U%U",
 963               format_white_space, indent,
 964               format_ip_adjacency_packet_data,
 965               vnm, t->adj_index,
 966               t->packet_data, sizeof (t->packet_data));
 967   return s;
 968 }
 969
 970 /* Common trace function for all ip6-forward next nodes. */
 971 void
 972 ip6_forward_next_trace (vlib_main_t * vm,
 973                         vlib_node_runtime_t * node,
 974                         vlib_frame_t * frame,
 975                         vlib_rx_or_tx_t which_adj_index)
 976 {
 977   u32 * from, n_left;
 978   ip6_main_t * im = &ip6_main;
 979
 980   n_left = frame->n_vectors;
 981   from = vlib_frame_vector_args (frame);
 982
 983   while (n_left >= 4)
 984     {
 985       u32 bi0, bi1;
 986       vlib_buffer_t * b0, * b1;
 987       ip6_forward_next_trace_t * t0, * t1;
 988
 989       /* Prefetch next iteration. */
 990       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 991       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 992
 993       bi0 = from[0];
 994       bi1 = from[1];
 995
 996       b0 = vlib_get_buffer (vm, bi0);
 997       b1 = vlib_get_buffer (vm, bi1);
 998
 999       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1000         {
1001           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1002           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1003           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1004           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1005               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1006               vec_elt (im->fib_index_by_sw_if_index,
1007                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1008
1009           clib_memcpy (t0->packet_data,
1010                   vlib_buffer_get_current (b0),
1011                   sizeof (t0->packet_data));
1012         }
1013       if (b1->flags & VLIB_BUFFER_IS_TRACED)
1014         {
1015           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1016           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1017           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1018           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
1019               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
1020               vec_elt (im->fib_index_by_sw_if_index,
1021                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
1022
1023           clib_memcpy (t1->packet_data,
1024                   vlib_buffer_get_current (b1),
1025                   sizeof (t1->packet_data));
1026         }
1027       from += 2;
1028       n_left -= 2;
1029     }
1030
1031   while (n_left >= 1)
1032     {
1033       u32 bi0;
1034       vlib_buffer_t * b0;
1035       ip6_forward_next_trace_t * t0;
1036
1037       bi0 = from[0];
1038
1039       b0 = vlib_get_buffer (vm, bi0);
1040
1041       if (b0->flags & VLIB_BUFFER_IS_TRACED)
1042         {
1043           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1044           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1045           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1046           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
1047               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
1048               vec_elt (im->fib_index_by_sw_if_index,
1049                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
1050
1051           clib_memcpy (t0->packet_data,
1052                   vlib_buffer_get_current (b0),
1053                   sizeof (t0->packet_data));
1054         }
1055       from += 1;
1056       n_left -= 1;
1057     }
1058 }
1059
1060 static uword
1061 ip6_drop_or_punt (vlib_main_t * vm,
1062                   vlib_node_runtime_t * node,
1063                   vlib_frame_t * frame,
1064                   ip6_error_t error_code)
1065 {
1066   u32 * buffers = vlib_frame_vector_args (frame);
1067   uword n_packets = frame->n_vectors;
1068
1069   vlib_error_drop_buffers (vm, node,
1070                            buffers,
1071                            /* stride */ 1,
1072                            n_packets,
1073                            /* next */ 0,
1074                            ip6_input_node.index,
1075                            error_code);
1076
1077   if (node->flags & VLIB_NODE_FLAG_TRACE)
1078     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1079
1080   return n_packets;
1081 }
1082
1083 static uword
1084 ip6_drop (vlib_main_t * vm,
1085           vlib_node_runtime_t * node,
1086           vlib_frame_t * frame)
1087 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1088
1089 static uword
1090 ip6_punt (vlib_main_t * vm,
1091           vlib_node_runtime_t * node,
1092           vlib_frame_t * frame)
1093 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1094
1095 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1096   .function = ip6_drop,
1097   .name = "ip6-drop",
1098   .vector_size = sizeof (u32),
1099
1100   .format_trace = format_ip6_forward_next_trace,
1101
1102   .n_next_nodes = 1,
1103   .next_nodes = {
1104     [0] = "error-drop",
1105   },
1106 };
1107
1108 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1109
1110 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1111   .function = ip6_punt,
1112   .name = "ip6-punt",
1113   .vector_size = sizeof (u32),
1114
1115   .format_trace = format_ip6_forward_next_trace,
1116
1117   .n_next_nodes = 1,
1118   .next_nodes = {
1119     [0] = "error-punt",
1120   },
1121 };
1122
1123 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1124
1125 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1126   .function = ip6_drop,
1127   .name = "ip6-multicast",
1128   .vector_size = sizeof (u32),
1129
1130   .format_trace = format_ip6_forward_next_trace,
1131
1132   .n_next_nodes = 1,
1133   .next_nodes = {
1134     [0] = "error-drop",
1135   },
1136 };
1137
1138 /* Compute TCP/UDP/ICMP6 checksum in software. */
1139 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1140 {
1141   ip_csum_t sum0;
1142   u16 sum16, payload_length_host_byte_order;
1143   u32 i, n_this_buffer, n_bytes_left;
1144   u32 headers_size = sizeof(ip0[0]);
1145   void * data_this_buffer;
1146
1147   ASSERT(bogus_lengthp);
1148   *bogus_lengthp = 0;
1149
1150   /* Initialize checksum with ip header. */
1151   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1152   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1153   data_this_buffer = (void *) (ip0 + 1);
1154
1155   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1156     {
1157       sum0 = ip_csum_with_carry (sum0,
1158                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1159       sum0 = ip_csum_with_carry (sum0,
1160                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1161     }
1162
1163   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1164   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1165     {
1166       u32  skip_bytes;
1167       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1168
1169       /* validate really icmp6 next */
1170       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1171
1172       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1173       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1174
1175       payload_length_host_byte_order  -= skip_bytes;
1176       headers_size += skip_bytes;
1177    }
1178
1179   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1180   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1181     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1182   while (1)
1183     {
1184       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1185       n_bytes_left -= n_this_buffer;
1186       if (n_bytes_left == 0)
1187         break;
1188
1189       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1190         {
1191           *bogus_lengthp = 1;
1192           return 0xfefe;
1193         }
1194       p0 = vlib_get_buffer (vm, p0->next_buffer);
1195       data_this_buffer = vlib_buffer_get_current (p0);
1196       n_this_buffer = p0->current_length;
1197     }
1198
1199   sum16 = ~ ip_csum_fold (sum0);
1200
1201   return sum16;
1202 }
1203
1204 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1205 {
1206   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1207   udp_header_t * udp0;
1208   u16 sum16;
1209   int bogus_length;
1210
1211   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1212   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1213           || ip0->protocol == IP_PROTOCOL_ICMP6
1214           || ip0->protocol == IP_PROTOCOL_UDP
1215           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1216
1217   udp0 = (void *) (ip0 + 1);
1218   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1219     {
1220       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1221                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1222       return p0->flags;
1223     }
1224
1225   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1226
1227   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1228                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1229
1230   return p0->flags;
1231 }
1232
1233 static uword
1234 ip6_local (vlib_main_t * vm,
1235            vlib_node_runtime_t * node,
1236            vlib_frame_t * frame)
1237 {
1238   ip6_main_t * im = &ip6_main;
1239   ip_lookup_main_t * lm = &im->lookup_main;
1240   ip_local_next_t next_index;
1241   u32 * from, * to_next, n_left_from, n_left_to_next;
1242   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1243
1244   from = vlib_frame_vector_args (frame);
1245   n_left_from = frame->n_vectors;
1246   next_index = node->cached_next_index;
1247
1248   if (node->flags & VLIB_NODE_FLAG_TRACE)
1249     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1250
1251   while (n_left_from > 0)
1252     {
1253       vlib_get_next_frame (vm, node, next_index,
1254                            to_next, n_left_to_next);
1255
1256       while (n_left_from >= 4 && n_left_to_next >= 2)
1257         {
1258           vlib_buffer_t * p0, * p1;
1259           ip6_header_t * ip0, * ip1;
1260           udp_header_t * udp0, * udp1;
1261           u32 pi0, ip_len0, udp_len0, flags0, next0;
1262           u32 pi1, ip_len1, udp_len1, flags1, next1;
1263           i32 len_diff0, len_diff1;
1264           u8 error0, type0, good_l4_checksum0;
1265           u8 error1, type1, good_l4_checksum1;
1266
1267           pi0 = to_next[0] = from[0];
1268           pi1 = to_next[1] = from[1];
1269           from += 2;
1270           n_left_from -= 2;
1271           to_next += 2;
1272           n_left_to_next -= 2;
1273
1274           p0 = vlib_get_buffer (vm, pi0);
1275           p1 = vlib_get_buffer (vm, pi1);
1276
1277           ip0 = vlib_buffer_get_current (p0);
1278           ip1 = vlib_buffer_get_current (p1);
1279
1280           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1281           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1282
1283           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1284           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1285
1286           flags0 = p0->flags;
1287           flags1 = p1->flags;
1288
1289           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1290           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1291
1292           udp0 = ip6_next_header (ip0);
1293           udp1 = ip6_next_header (ip1);
1294
1295           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1296           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1297           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1298
1299           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1300           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1301
1302           /* Verify UDP length. */
1303           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1304           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1305           udp_len0 = clib_net_to_host_u16 (udp0->length);
1306           udp_len1 = clib_net_to_host_u16 (udp1->length);
1307
1308           len_diff0 = ip_len0 - udp_len0;
1309           len_diff1 = ip_len1 - udp_len1;
1310
1311           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1312           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1313
1314           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1315                              && ! good_l4_checksum0
1316                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1317             {
1318               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1319               good_l4_checksum0 =
1320                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1321             }
1322           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1323                              && ! good_l4_checksum1
1324                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1325             {
1326               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1327               good_l4_checksum1 =
1328                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1329             }
1330
1331           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1332
1333           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1334           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1335
1336           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1337           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1338           error0 = (! good_l4_checksum0
1339                     ? IP6_ERROR_UDP_CHECKSUM + type0
1340                     : error0);
1341           error1 = (! good_l4_checksum1
1342                     ? IP6_ERROR_UDP_CHECKSUM + type1
1343                     : error1);
1344
1345           /* Drop packets from unroutable hosts. */
1346           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1347           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1348               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1349               !ip6_address_is_link_local_unicast(&ip0->src_address))
1350             {
1351               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1352               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1353                         ? IP6_ERROR_SRC_LOOKUP_MISS
1354                         : error0);
1355             }
1356           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1357               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1358               !ip6_address_is_link_local_unicast(&ip1->src_address))
1359             {
1360               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1361               error1 = (ADJ_INDEX_INVALID == src_adj_index1
1362                         ? IP6_ERROR_SRC_LOOKUP_MISS
1363                         : error1);
1364             }
1365
1366           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1367           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1368
1369           p0->error = error_node->errors[error0];
1370           p1->error = error_node->errors[error1];
1371
1372           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1373                                            to_next, n_left_to_next,
1374                                            pi0, pi1, next0, next1);
1375         }
1376
1377       while (n_left_from > 0 && n_left_to_next > 0)
1378         {
1379           vlib_buffer_t * p0;
1380           ip6_header_t * ip0;
1381           udp_header_t * udp0;
1382           u32 pi0, ip_len0, udp_len0, flags0, next0;
1383           i32 len_diff0;
1384           u8 error0, type0, good_l4_checksum0;
1385
1386           pi0 = to_next[0] = from[0];
1387           from += 1;
1388           n_left_from -= 1;
1389           to_next += 1;
1390           n_left_to_next -= 1;
1391
1392           p0 = vlib_get_buffer (vm, pi0);
1393
1394           ip0 = vlib_buffer_get_current (p0);
1395
1396           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1397           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1398
1399           flags0 = p0->flags;
1400
1401           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1402
1403           udp0 = ip6_next_header (ip0);
1404
1405           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1406           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1407
1408           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1409
1410           /* Verify UDP length. */
1411           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1412           udp_len0 = clib_net_to_host_u16 (udp0->length);
1413
1414           len_diff0 = ip_len0 - udp_len0;
1415
1416           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1417
1418           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1419                              && ! good_l4_checksum0
1420                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1421             {
1422               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1423               good_l4_checksum0 =
1424                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1425             }
1426
1427           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1428
1429           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1430
1431           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1432           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1433           error0 = (! good_l4_checksum0
1434                     ? IP6_ERROR_UDP_CHECKSUM + type0
1435                     : error0);
1436
1437           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1438           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1439               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1440               !ip6_address_is_link_local_unicast(&ip0->src_address))
1441             {
1442               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1443               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1444                         ? IP6_ERROR_SRC_LOOKUP_MISS
1445                         : error0);
1446             }
1447
1448           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1449
1450           p0->error = error_node->errors[error0];
1451
1452           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1453                                            to_next, n_left_to_next,
1454                                            pi0, next0);
1455         }
1456
1457       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1458     }
1459
1460   return frame->n_vectors;
1461 }
1462
1463 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1464   .function = ip6_local,
1465   .name = "ip6-local",
1466   .vector_size = sizeof (u32),
1467
1468   .format_trace = format_ip6_forward_next_trace,
1469
1470   .n_next_nodes = IP_LOCAL_N_NEXT,
1471   .next_nodes = {
1472     [IP_LOCAL_NEXT_DROP] = "error-drop",
1473     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1474     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1475     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1476   },
1477 };
1478
1479 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
1480
1481 void ip6_register_protocol (u32 protocol, u32 node_index)
1482 {
1483   vlib_main_t * vm = vlib_get_main();
1484   ip6_main_t * im = &ip6_main;
1485   ip_lookup_main_t * lm = &im->lookup_main;
1486
1487   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1488   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1489 }
1490
1491 typedef enum {
1492   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1493   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1494   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1495 } ip6_discover_neighbor_next_t;
1496
1497 typedef enum {
1498   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1499   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1500   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
1501 } ip6_discover_neighbor_error_t;
1502
1503 static uword
1504 ip6_discover_neighbor_inline (vlib_main_t * vm,
1505                               vlib_node_runtime_t * node,
1506                               vlib_frame_t * frame,
1507                               int is_glean)
1508 {
1509   vnet_main_t * vnm = vnet_get_main();
1510   ip6_main_t * im = &ip6_main;
1511   ip_lookup_main_t * lm = &im->lookup_main;
1512   u32 * from, * to_next_drop;
1513   uword n_left_from, n_left_to_next_drop;
1514   static f64 time_last_seed_change = -1e100;
1515   static u32 hash_seeds[3];
1516   static uword hash_bitmap[256 / BITS (uword)];
1517   f64 time_now;
1518   int bogus_length;
1519
1520   if (node->flags & VLIB_NODE_FLAG_TRACE)
1521     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1522
1523   time_now = vlib_time_now (vm);
1524   if (time_now - time_last_seed_change > 1e-3)
1525     {
1526       uword i;
1527       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1528                                              sizeof (hash_seeds));
1529       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1530         hash_seeds[i] = r[i];
1531
1532       /* Mark all hash keys as been not-seen before. */
1533       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1534         hash_bitmap[i] = 0;
1535
1536       time_last_seed_change = time_now;
1537     }
1538
1539   from = vlib_frame_vector_args (frame);
1540   n_left_from = frame->n_vectors;
1541
1542   while (n_left_from > 0)
1543     {
1544       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1545                            to_next_drop, n_left_to_next_drop);
1546
1547       while (n_left_from > 0 && n_left_to_next_drop > 0)
1548         {
1549           vlib_buffer_t * p0;
1550           ip6_header_t * ip0;
1551           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1552           uword bm0;
1553           ip_adjacency_t * adj0;
1554           vnet_hw_interface_t * hw_if0;
1555           u32 next0;
1556
1557           pi0 = from[0];
1558
1559           p0 = vlib_get_buffer (vm, pi0);
1560
1561           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1562
1563           ip0 = vlib_buffer_get_current (p0);
1564
1565           adj0 = ip_get_adjacency (lm, adj_index0);
1566
1567           if (!is_glean)
1568             {
1569               ip0->dst_address.as_u64[0] = adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
1570               ip0->dst_address.as_u64[1] = adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
1571             }
1572
1573           a0 = hash_seeds[0];
1574           b0 = hash_seeds[1];
1575           c0 = hash_seeds[2];
1576
1577           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1578           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1579
1580           a0 ^= sw_if_index0;
1581           b0 ^= ip0->dst_address.as_u32[0];
1582           c0 ^= ip0->dst_address.as_u32[1];
1583
1584           hash_v3_mix32 (a0, b0, c0);
1585
1586           b0 ^= ip0->dst_address.as_u32[2];
1587           c0 ^= ip0->dst_address.as_u32[3];
1588
1589           hash_v3_finalize32 (a0, b0, c0);
1590
1591           c0 &= BITS (hash_bitmap) - 1;
1592           c0 = c0 / BITS (uword);
1593           m0 = (uword) 1 << (c0 % BITS (uword));
1594
1595           bm0 = hash_bitmap[c0];
1596           drop0 = (bm0 & m0) != 0;
1597
1598           /* Mark it as seen. */
1599           hash_bitmap[c0] = bm0 | m0;
1600
1601           from += 1;
1602           n_left_from -= 1;
1603           to_next_drop[0] = pi0;
1604           to_next_drop += 1;
1605           n_left_to_next_drop -= 1;
1606
1607           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1608
1609           /* If the interface is link-down, drop the pkt */
1610           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1611             drop0 = 1;
1612
1613           p0->error =
1614             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1615                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1616           if (drop0)
1617             continue;
1618
1619           /*
1620            * the adj has been updated to a rewrite but the node the DPO that got
1621            * us here hasn't - yet. no big deal. we'll drop while we wait.
1622            */
1623           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1624             continue;
1625
1626           {
1627             u32 bi0 = 0;
1628             icmp6_neighbor_solicitation_header_t * h0;
1629             vlib_buffer_t * b0;
1630
1631             h0 = vlib_packet_template_get_packet
1632               (vm, &im->discover_neighbor_packet_template, &bi0);
1633
1634             /*
1635              * Build ethernet header.
1636              * Choose source address based on destination lookup
1637              * adjacency.
1638              */
1639             if (ip6_src_address_for_packet (lm,
1640                                             sw_if_index0,
1641                                             &h0->ip.src_address))
1642               {
1643                 /* There is no address on the interface */
1644                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
1645                 vlib_buffer_free(vm, &bi0, 1);
1646                 continue;
1647               }
1648
1649             /*
1650              * Destination address is a solicited node multicast address.
1651              * We need to fill in
1652              * the low 24 bits with low 24 bits of target's address.
1653              */
1654             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1655             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1656             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1657
1658             h0->neighbor.target_address = ip0->dst_address;
1659
1660             clib_memcpy (h0->link_layer_option.ethernet_address,
1661                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
1662
1663             /* $$$$ appears we need this; why is the checksum non-zero? */
1664             h0->neighbor.icmp.checksum = 0;
1665             h0->neighbor.icmp.checksum =
1666               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1667                                                  &bogus_length);
1668
1669             ASSERT (bogus_length == 0);
1670
1671             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1672             b0 = vlib_get_buffer (vm, bi0);
1673             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1674               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1675
1676             /* Add rewrite/encap string. */
1677             vnet_rewrite_one_header (adj0[0], h0,
1678                                      sizeof (ethernet_header_t));
1679             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1680
1681             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1682
1683             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1684           }
1685         }
1686
1687       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1688                            n_left_to_next_drop);
1689     }
1690
1691   return frame->n_vectors;
1692 }
1693
1694 static uword
1695 ip6_discover_neighbor (vlib_main_t * vm,
1696                        vlib_node_runtime_t * node,
1697                        vlib_frame_t * frame)
1698 {
1699     return (ip6_discover_neighbor_inline(vm, node, frame, 0));
1700 }
1701
1702 static uword
1703 ip6_glean (vlib_main_t * vm,
1704            vlib_node_runtime_t * node,
1705            vlib_frame_t * frame)
1706 {
1707     return (ip6_discover_neighbor_inline(vm, node, frame, 1));
1708 }
1709
1710 static char * ip6_discover_neighbor_error_strings[] = {
1711   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1712   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
1713   = "neighbor solicitations sent",
1714   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
1715     = "no source address for ND solicitation",
1716 };
1717
1718 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
1719   .function = ip6_discover_neighbor,
1720   .name = "ip6-discover-neighbor",
1721   .vector_size = sizeof (u32),
1722
1723   .format_trace = format_ip6_forward_next_trace,
1724
1725   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1726   .error_strings = ip6_discover_neighbor_error_strings,
1727
1728   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1729   .next_nodes = {
1730     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1731     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1732   },
1733 };
1734
1735 VLIB_REGISTER_NODE (ip6_glean_node) = {
1736   .function = ip6_glean,
1737   .name = "ip6-glean",
1738   .vector_size = sizeof (u32),
1739
1740   .format_trace = format_ip6_forward_next_trace,
1741
1742   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1743   .error_strings = ip6_discover_neighbor_error_strings,
1744
1745   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1746   .next_nodes = {
1747     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1748     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1749   },
1750 };
1751
1752 clib_error_t *
1753 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
1754 {
1755   vnet_main_t * vnm = vnet_get_main();
1756   ip6_main_t * im = &ip6_main;
1757   icmp6_neighbor_solicitation_header_t * h;
1758   ip6_address_t * src;
1759   ip_interface_address_t * ia;
1760   ip_adjacency_t * adj;
1761   vnet_hw_interface_t * hi;
1762   vnet_sw_interface_t * si;
1763   vlib_buffer_t * b;
1764   u32 bi = 0;
1765   int bogus_length;
1766
1767   si = vnet_get_sw_interface (vnm, sw_if_index);
1768
1769   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1770     {
1771       return clib_error_return (0, "%U: interface %U down",
1772                                 format_ip6_address, dst,
1773                                 format_vnet_sw_if_index_name, vnm,
1774                                 sw_if_index);
1775     }
1776
1777   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1778   if (! src)
1779     {
1780       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1781       return clib_error_return
1782         (0, "no matching interface address for destination %U (interface %U)",
1783          format_ip6_address, dst,
1784          format_vnet_sw_if_index_name, vnm, sw_if_index);
1785     }
1786
1787   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
1788
1789   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1790
1791   /* Destination address is a solicited node multicast address.  We need to fill in
1792      the low 24 bits with low 24 bits of target's address. */
1793   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1794   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1795   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1796
1797   h->ip.src_address = src[0];
1798   h->neighbor.target_address = dst[0];
1799
1800   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
1801
1802   h->neighbor.icmp.checksum =
1803     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1804   ASSERT(bogus_length == 0);
1805
1806   b = vlib_get_buffer (vm, bi);
1807   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1808
1809   /* Add encapsulation string for software interface (e.g. ethernet header). */
1810   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
1811   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1812   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1813
1814   {
1815     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
1816     u32 * to_next = vlib_frame_vector_args (f);
1817     to_next[0] = bi;
1818     f->n_vectors = 1;
1819     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1820   }
1821
1822   return /* no error */ 0;
1823 }
1824
1825 typedef enum {
1826   IP6_REWRITE_NEXT_DROP,
1827   IP6_REWRITE_NEXT_ICMP_ERROR,
1828 } ip6_rewrite_next_t;
1829
1830 always_inline uword
1831 ip6_rewrite_inline (vlib_main_t * vm,
1832                     vlib_node_runtime_t * node,
1833                     vlib_frame_t * frame,
1834                     int rewrite_for_locally_received_packets,
1835                     int is_midchain)
1836 {
1837   ip_lookup_main_t * lm = &ip6_main.lookup_main;
1838   u32 * from = vlib_frame_vector_args (frame);
1839   u32 n_left_from, n_left_to_next, * to_next, next_index;
1840   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1841   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
1842
1843   n_left_from = frame->n_vectors;
1844   next_index = node->cached_next_index;
1845   u32 cpu_index = os_get_cpu_number();
1846
1847   while (n_left_from > 0)
1848     {
1849       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1850
1851       while (n_left_from >= 4 && n_left_to_next >= 2)
1852         {
1853           ip_adjacency_t * adj0, * adj1;
1854           vlib_buffer_t * p0, * p1;
1855           ip6_header_t * ip0, * ip1;
1856           u32 pi0, rw_len0, next0, error0, adj_index0;
1857           u32 pi1, rw_len1, next1, error1, adj_index1;
1858           u32 tx_sw_if_index0, tx_sw_if_index1;
1859
1860           /* Prefetch next iteration. */
1861           {
1862             vlib_buffer_t * p2, * p3;
1863
1864             p2 = vlib_get_buffer (vm, from[2]);
1865             p3 = vlib_get_buffer (vm, from[3]);
1866
1867             vlib_prefetch_buffer_header (p2, LOAD);
1868             vlib_prefetch_buffer_header (p3, LOAD);
1869
1870             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1871             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1872
1873             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1874             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1875           }
1876
1877           pi0 = to_next[0] = from[0];
1878           pi1 = to_next[1] = from[1];
1879
1880           from += 2;
1881           n_left_from -= 2;
1882           to_next += 2;
1883           n_left_to_next -= 2;
1884
1885           p0 = vlib_get_buffer (vm, pi0);
1886           p1 = vlib_get_buffer (vm, pi1);
1887
1888           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
1889           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
1890
1891           /* We should never rewrite a pkt using the MISS adjacency */
1892           ASSERT(adj_index0 && adj_index1);
1893
1894           ip0 = vlib_buffer_get_current (p0);
1895           ip1 = vlib_buffer_get_current (p1);
1896
1897           error0 = error1 = IP6_ERROR_NONE;
1898           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1899
1900           if (! rewrite_for_locally_received_packets)
1901             {
1902               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
1903
1904               /* Input node should have reject packets with hop limit 0. */
1905               ASSERT (ip0->hop_limit > 0);
1906               ASSERT (ip1->hop_limit > 0);
1907
1908               hop_limit0 -= 1;
1909               hop_limit1 -= 1;
1910
1911               ip0->hop_limit = hop_limit0;
1912               ip1->hop_limit = hop_limit1;
1913
1914               /*
1915                * If the hop count drops below 1 when forwarding, generate
1916                * an ICMP response.
1917                */
1918               if (PREDICT_FALSE(hop_limit0 <= 0))
1919                 {
1920                   error0 = IP6_ERROR_TIME_EXPIRED;
1921                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1922                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
1923                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
1924                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1925                 }
1926               if (PREDICT_FALSE(hop_limit1 <= 0))
1927                 {
1928                   error1 = IP6_ERROR_TIME_EXPIRED;
1929                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1930                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
1931                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
1932                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1933                 }
1934             }
1935
1936           adj0 = ip_get_adjacency (lm, adj_index0);
1937           adj1 = ip_get_adjacency (lm, adj_index1);
1938
1939           rw_len0 = adj0[0].rewrite_header.data_bytes;
1940           rw_len1 = adj1[0].rewrite_header.data_bytes;
1941           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
1942           vnet_buffer(p1)->ip.save_rewrite_length = rw_len1;
1943
1944           vlib_increment_combined_counter (&adjacency_counters,
1945                                            cpu_index,
1946                                            adj_index0,
1947                                            /* packet increment */ 0,
1948                                            /* byte increment */ rw_len0);
1949           vlib_increment_combined_counter (&adjacency_counters,
1950                                            cpu_index,
1951                                            adj_index1,
1952                                            /* packet increment */ 0,
1953                                            /* byte increment */ rw_len1);
1954
1955           /* Check MTU of outgoing interface. */
1956           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
1957                     ? IP6_ERROR_MTU_EXCEEDED
1958                     : error0);
1959           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
1960                     ? IP6_ERROR_MTU_EXCEEDED
1961                     : error1);
1962
1963           /* Don't adjust the buffer for hop count issue; icmp-error node
1964            * wants to see the IP headerr */
1965           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
1966             {
1967               p0->current_data -= rw_len0;
1968               p0->current_length += rw_len0;
1969
1970               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1971               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
1972                   tx_sw_if_index0;
1973               next0 = adj0[0].rewrite_header.next_index;
1974
1975               vnet_feature_arc_start(lm->output_feature_arc_index,
1976                                      tx_sw_if_index0, &next0, p0);
1977             }
1978           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
1979             {
1980               p1->current_data -= rw_len1;
1981               p1->current_length += rw_len1;
1982
1983               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1984               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
1985                   tx_sw_if_index1;
1986               next1 = adj1[0].rewrite_header.next_index;
1987
1988               vnet_feature_arc_start(lm->output_feature_arc_index,
1989                                      tx_sw_if_index1, &next1, p1);
1990             }
1991
1992           /* Guess we are only writing on simple Ethernet header. */
1993           vnet_rewrite_two_headers (adj0[0], adj1[0],
1994                                     ip0, ip1,
1995                                     sizeof (ethernet_header_t));
1996
1997           if (is_midchain)
1998           {
1999               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2000               adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
2001           }
2002
2003           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2004                                            to_next, n_left_to_next,
2005                                            pi0, pi1, next0, next1);
2006         }
2007
2008       while (n_left_from > 0 && n_left_to_next > 0)
2009         {
2010           ip_adjacency_t * adj0;
2011           vlib_buffer_t * p0;
2012           ip6_header_t * ip0;
2013           u32 pi0, rw_len0;
2014           u32 adj_index0, next0, error0;
2015           u32 tx_sw_if_index0;
2016
2017           pi0 = to_next[0] = from[0];
2018
2019           p0 = vlib_get_buffer (vm, pi0);
2020
2021           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2022
2023           /* We should never rewrite a pkt using the MISS adjacency */
2024           ASSERT(adj_index0);
2025
2026           adj0 = ip_get_adjacency (lm, adj_index0);
2027
2028           ip0 = vlib_buffer_get_current (p0);
2029
2030           error0 = IP6_ERROR_NONE;
2031           next0 = IP6_REWRITE_NEXT_DROP;
2032
2033           /* Check hop limit */
2034           if (! rewrite_for_locally_received_packets)
2035             {
2036               i32 hop_limit0 = ip0->hop_limit;
2037
2038               ASSERT (ip0->hop_limit > 0);
2039
2040               hop_limit0 -= 1;
2041
2042               ip0->hop_limit = hop_limit0;
2043
2044               if (PREDICT_FALSE(hop_limit0 <= 0))
2045                 {
2046                   /*
2047                    * If the hop count drops below 1 when forwarding, generate
2048                    * an ICMP response.
2049                    */
2050                   error0 = IP6_ERROR_TIME_EXPIRED;
2051                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2052                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2053                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2054                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2055                 }
2056             }
2057
2058           /* Guess we are only writing on simple Ethernet header. */
2059           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2060
2061           /* Update packet buffer attributes/set output interface. */
2062           rw_len0 = adj0[0].rewrite_header.data_bytes;
2063           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2064
2065           vlib_increment_combined_counter (&adjacency_counters,
2066                                            cpu_index,
2067                                            adj_index0,
2068                                            /* packet increment */ 0,
2069                                            /* byte increment */ rw_len0);
2070
2071           /* Check MTU of outgoing interface. */
2072           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2073                     ? IP6_ERROR_MTU_EXCEEDED
2074                     : error0);
2075
2076           /* Don't adjust the buffer for hop count issue; icmp-error node
2077            * wants to see the IP headerr */
2078           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2079             {
2080               p0->current_data -= rw_len0;
2081               p0->current_length += rw_len0;
2082
2083               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2084
2085               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2086               next0 = adj0[0].rewrite_header.next_index;
2087
2088               vnet_feature_arc_start(lm->output_feature_arc_index,
2089                                      tx_sw_if_index0, &next0, p0);
2090             }
2091
2092           if (is_midchain)
2093           {
2094               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2095           }
2096
2097           p0->error = error_node->errors[error0];
2098
2099           from += 1;
2100           n_left_from -= 1;
2101           to_next += 1;
2102           n_left_to_next -= 1;
2103
2104           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2105                                            to_next, n_left_to_next,
2106                                            pi0, next0);
2107         }
2108
2109       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2110     }
2111
2112   /* Need to do trace after rewrites to pick up new packet data. */
2113   if (node->flags & VLIB_NODE_FLAG_TRACE)
2114     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2115
2116   return frame->n_vectors;
2117 }
2118
2119 static uword
2120 ip6_rewrite_transit (vlib_main_t * vm,
2121                      vlib_node_runtime_t * node,
2122                      vlib_frame_t * frame)
2123 {
2124   return ip6_rewrite_inline (vm, node, frame,
2125                              /* rewrite_for_locally_received_packets */ 0,
2126                              /* midchain */ 0);
2127 }
2128
2129 static uword
2130 ip6_rewrite_local (vlib_main_t * vm,
2131                    vlib_node_runtime_t * node,
2132                    vlib_frame_t * frame)
2133 {
2134   return ip6_rewrite_inline (vm, node, frame,
2135                              /* rewrite_for_locally_received_packets */ 1,
2136                              /* midchain */ 0);
2137 }
2138
2139 static uword
2140 ip6_midchain (vlib_main_t * vm,
2141               vlib_node_runtime_t * node,
2142               vlib_frame_t * frame)
2143 {
2144   return ip6_rewrite_inline (vm, node, frame,
2145                              /* rewrite_for_locally_received_packets */ 0,
2146                              /* midchain */ 1);
2147 }
2148
2149 VLIB_REGISTER_NODE (ip6_midchain_node) = {
2150   .function = ip6_midchain,
2151   .name = "ip6-midchain",
2152   .vector_size = sizeof (u32),
2153
2154   .format_trace = format_ip6_forward_next_trace,
2155
2156   .sibling_of = "ip6-rewrite",
2157 };
2158
2159 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain)
2160
2161 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2162   .function = ip6_rewrite_transit,
2163   .name = "ip6-rewrite",
2164   .vector_size = sizeof (u32),
2165
2166   .format_trace = format_ip6_rewrite_trace,
2167
2168   .n_next_nodes = 2,
2169   .next_nodes = {
2170     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2171     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2172   },
2173 };
2174
2175 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit);
2176
2177 VLIB_REGISTER_NODE (ip6_rewrite_local_node) = {
2178   .function = ip6_rewrite_local,
2179   .name = "ip6-rewrite-local",
2180   .vector_size = sizeof (u32),
2181
2182   .sibling_of = "ip6-rewrite",
2183
2184   .format_trace = format_ip6_rewrite_trace,
2185
2186   .n_next_nodes = 0,
2187 };
2188
2189 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local);
2190
2191 /*
2192  * Hop-by-Hop handling
2193  */
2194
2195 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2196
2197 #define foreach_ip6_hop_by_hop_error \
2198 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2199 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2200 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2201
2202 typedef enum {
2203 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2204   foreach_ip6_hop_by_hop_error
2205 #undef _
2206   IP6_HOP_BY_HOP_N_ERROR,
2207 } ip6_hop_by_hop_error_t;
2208
2209 /*
2210  * Primary h-b-h handler trace support
2211  * We work pretty hard on the problem for obvious reasons
2212  */
2213 typedef struct {
2214   u32 next_index;
2215   u32 trace_len;
2216   u8 option_data[256];
2217 } ip6_hop_by_hop_trace_t;
2218
2219 vlib_node_registration_t ip6_hop_by_hop_node;
2220
2221 static char * ip6_hop_by_hop_error_strings[] = {
2222 #define _(sym,string) string,
2223   foreach_ip6_hop_by_hop_error
2224 #undef _
2225 };
2226
2227 static u8 *
2228 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2229 {
2230   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2231   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2232   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2233   ip6_hop_by_hop_header_t *hbh0;
2234   ip6_hop_by_hop_option_t *opt0, *limit0;
2235   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2236
2237   u8 type0;
2238
2239   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2240
2241   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2242               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2243
2244   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2245   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2246
2247   while (opt0 < limit0) {
2248     type0 = opt0->type;
2249     switch (type0) {
2250     case 0: /* Pad, just stop */
2251       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2252       break;
2253
2254     default:
2255       if (hm->trace[type0]) {
2256         s = (*hm->trace[type0])(s, opt0);
2257       } else {
2258         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2259       }
2260       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2261       break;
2262     }
2263   }
2264   return s;
2265 }
2266
2267 always_inline u8 ip6_scan_hbh_options (
2268                                        vlib_buffer_t * b0,
2269                                        ip6_header_t *ip0,
2270                                        ip6_hop_by_hop_header_t *hbh0,
2271                                        ip6_hop_by_hop_option_t *opt0,
2272                                        ip6_hop_by_hop_option_t *limit0,
2273                                        u32 *next0)
2274 {
2275   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2276   u8 type0;
2277   u8 error0 = 0;
2278
2279   while (opt0 < limit0)
2280     {
2281       type0 = opt0->type;
2282       switch (type0)
2283         {
2284         case 0: /* Pad1 */
2285           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2286           continue;
2287         case 1: /* PadN */
2288           break;
2289         default:
2290           if (hm->options[type0])
2291             {
2292               if ((*hm->options[type0])(b0, ip0, opt0) < 0)
2293                 {
2294                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2295                   return(error0);
2296                 }
2297             }
2298           else
2299             {
2300               /* Unrecognized mandatory option, check the two high order bits */
2301               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2302                 {
2303                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2304                   break;
2305                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2306                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2307                   *next0 = IP_LOOKUP_NEXT_DROP;
2308                   break;
2309                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2310                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2311                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2312                   icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2313                                               ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2314                   break;
2315                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2316                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2317                   if (!ip6_address_is_multicast(&ip0->dst_address))
2318                     {
2319                       *next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2320                       icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2321                                                   ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2322                     }
2323                   else
2324                     {
2325                       *next0 =  IP_LOOKUP_NEXT_DROP;
2326                     }
2327                   break;
2328                 }
2329               return(error0);
2330             }
2331         }
2332       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2333     }
2334   return(error0);
2335 }
2336
2337 /*
2338  * Process the Hop-by-Hop Options header
2339  */
2340 static uword
2341 ip6_hop_by_hop (vlib_main_t * vm,
2342                 vlib_node_runtime_t * node,
2343                 vlib_frame_t * frame)
2344 {
2345   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2346   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2347   u32 n_left_from, *from, *to_next;
2348   ip_lookup_next_t next_index;
2349   ip6_main_t * im = &ip6_main;
2350   ip_lookup_main_t *lm = &im->lookup_main;
2351
2352   from = vlib_frame_vector_args (frame);
2353   n_left_from = frame->n_vectors;
2354   next_index = node->cached_next_index;
2355
2356   while (n_left_from > 0) {
2357     u32 n_left_to_next;
2358
2359     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2360
2361     while (n_left_from >= 4 && n_left_to_next >= 2) {
2362       u32 bi0, bi1;
2363       vlib_buffer_t * b0, *b1;
2364       u32 next0, next1;
2365       ip6_header_t * ip0, *ip1;
2366       ip6_hop_by_hop_header_t *hbh0, *hbh1;
2367       ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2368       u8 error0 = 0, error1 = 0;
2369
2370       /* Prefetch next iteration. */
2371       {
2372         vlib_buffer_t * p2, * p3;
2373
2374         p2 = vlib_get_buffer (vm, from[2]);
2375         p3 = vlib_get_buffer (vm, from[3]);
2376
2377         vlib_prefetch_buffer_header (p2, LOAD);
2378         vlib_prefetch_buffer_header (p3, LOAD);
2379
2380         CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2381         CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2382       }
2383
2384       /* Speculatively enqueue b0, b1 to the current next frame */
2385       to_next[0] = bi0 = from[0];
2386       to_next[1] = bi1 = from[1];
2387       from += 2;
2388       to_next += 2;
2389       n_left_from -= 2;
2390       n_left_to_next -= 2;
2391
2392       b0 = vlib_get_buffer (vm, bi0);
2393       b1 = vlib_get_buffer (vm, bi1);
2394
2395       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2396       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2397       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2398       u32 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
2399       ip_adjacency_t *adj1 = ip_get_adjacency(lm, adj_index1);
2400
2401       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2402       next0 = adj0->lookup_next_index;
2403       next1 = adj1->lookup_next_index;
2404
2405       ip0 = vlib_buffer_get_current (b0);
2406       ip1 = vlib_buffer_get_current (b1);
2407       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2408       hbh1 = (ip6_hop_by_hop_header_t *)(ip1+1);
2409       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2410       opt1 = (ip6_hop_by_hop_option_t *)(hbh1+1);
2411       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2412       limit1 = (ip6_hop_by_hop_option_t *)((u8 *)hbh1 + ((hbh1->length + 1) << 3));
2413
2414       /*
2415        * Basic validity checks
2416        */
2417       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2418         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2419         next0 = IP_LOOKUP_NEXT_DROP;
2420         goto outdual;
2421       }
2422       /* Scan the set of h-b-h options, process ones that we understand */
2423       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2424
2425       if ((hbh1->length + 1) << 3 > clib_net_to_host_u16(ip1->payload_length)) {
2426         error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2427         next1 = IP_LOOKUP_NEXT_DROP;
2428         goto outdual;
2429       }
2430       /* Scan the set of h-b-h options, process ones that we understand */
2431       error1 = ip6_scan_hbh_options(b1,ip1,hbh1,opt1,limit1, &next1);
2432
2433     outdual:
2434       /* Has the classifier flagged this buffer for special treatment? */
2435       if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2436         next0 = hm->next_override;
2437
2438       /* Has the classifier flagged this buffer for special treatment? */
2439       if (PREDICT_FALSE((error1 == 0) && (vnet_buffer(b1)->l2_classify.opaque_index & OI_DECAP)))
2440         next1 = hm->next_override;
2441
2442       if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
2443         {
2444           if (b0->flags & VLIB_BUFFER_IS_TRACED) {
2445             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2446             u32 trace_len = (hbh0->length + 1) << 3;
2447             t->next_index = next0;
2448             /* Capture the h-b-h option verbatim */
2449             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2450             t->trace_len = trace_len;
2451             clib_memcpy(t->option_data, hbh0, trace_len);
2452           }
2453           if (b1->flags & VLIB_BUFFER_IS_TRACED) {
2454             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b1, sizeof (*t));
2455             u32 trace_len = (hbh1->length + 1) << 3;
2456             t->next_index = next1;
2457             /* Capture the h-b-h option verbatim */
2458             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2459             t->trace_len = trace_len;
2460             clib_memcpy(t->option_data, hbh1, trace_len);
2461           }
2462
2463         }
2464
2465       b0->error = error_node->errors[error0];
2466       b1->error = error_node->errors[error1];
2467
2468       /* verify speculative enqueue, maybe switch current next frame */
2469       vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0,
2470                                        bi1,next0, next1);
2471     }
2472
2473     while (n_left_from > 0 && n_left_to_next > 0) {
2474       u32 bi0;
2475       vlib_buffer_t * b0;
2476       u32 next0;
2477       ip6_header_t * ip0;
2478       ip6_hop_by_hop_header_t *hbh0;
2479       ip6_hop_by_hop_option_t *opt0, *limit0;
2480       u8 error0 = 0;
2481
2482       /* Speculatively enqueue b0 to the current next frame */
2483       bi0 = from[0];
2484       to_next[0] = bi0;
2485       from += 1;
2486       to_next += 1;
2487       n_left_from -= 1;
2488       n_left_to_next -= 1;
2489
2490       b0 = vlib_get_buffer (vm, bi0);
2491       /*
2492        * Default use the next_index from the adjacency.
2493        * A HBH option rarely redirects to a different node
2494        */
2495       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2496       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2497       next0 = adj0->lookup_next_index;
2498
2499       ip0 = vlib_buffer_get_current (b0);
2500       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2501       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2502       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2503
2504       /*
2505        * Basic validity checks
2506        */
2507       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2508         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2509         next0 = IP_LOOKUP_NEXT_DROP;
2510         goto out0;
2511       }
2512
2513       /* Scan the set of h-b-h options, process ones that we understand */
2514       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2515
2516     out0:
2517       /* Has the classifier flagged this buffer for special treatment? */
2518     if (PREDICT_FALSE((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index & OI_DECAP)))
2519         next0 = hm->next_override;
2520
2521       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2522         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2523         u32 trace_len = (hbh0->length + 1) << 3;
2524         t->next_index = next0;
2525         /* Capture the h-b-h option verbatim */
2526         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2527         t->trace_len = trace_len;
2528         clib_memcpy(t->option_data, hbh0, trace_len);
2529       }
2530
2531       b0->error = error_node->errors[error0];
2532
2533       /* verify speculative enqueue, maybe switch current next frame */
2534       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2535     }
2536     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2537   }
2538   return frame->n_vectors;
2539 }
2540
2541 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2542   .function = ip6_hop_by_hop,
2543   .name = "ip6-hop-by-hop",
2544   .sibling_of = "ip6-lookup",
2545   .vector_size = sizeof (u32),
2546   .format_trace = format_ip6_hop_by_hop_trace,
2547   .type = VLIB_NODE_TYPE_INTERNAL,
2548   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2549   .error_strings = ip6_hop_by_hop_error_strings,
2550   .n_next_nodes = 0,
2551 };
2552
2553 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2554
2555 static clib_error_t *
2556 ip6_hop_by_hop_init (vlib_main_t * vm)
2557 {
2558   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2559   memset(hm->options, 0, sizeof(hm->options));
2560   memset(hm->trace, 0, sizeof(hm->trace));
2561   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2562   return (0);
2563 }
2564
2565 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2566
2567 void ip6_hbh_set_next_override (uword next)
2568 {
2569   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2570
2571   hm->next_override = next;
2572 }
2573
2574 int
2575 ip6_hbh_register_option (u8 option,
2576                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2577                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2578 {
2579   ip6_main_t * im = &ip6_main;
2580   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2581
2582   ASSERT (option < ARRAY_LEN (hm->options));
2583
2584   /* Already registered */
2585   if (hm->options[option])
2586     return (-1);
2587
2588   hm->options[option] = options;
2589   hm->trace[option] = trace;
2590
2591   /* Set global variable */
2592   im->hbh_enabled = 1;
2593
2594   return (0);
2595 }
2596
2597 int
2598 ip6_hbh_unregister_option (u8 option)
2599 {
2600   ip6_main_t * im = &ip6_main;
2601   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2602
2603   ASSERT (option < ARRAY_LEN (hm->options));
2604
2605   /* Not registered */
2606   if (!hm->options[option])
2607     return (-1);
2608
2609   hm->options[option] = NULL;
2610   hm->trace[option] = NULL;
2611
2612   /* Disable global knob if this was the last option configured */
2613   int i;
2614   bool found = false;
2615   for (i = 0; i < 256; i++) {
2616     if (hm->options[option]) {
2617       found = true;
2618       break;
2619     }
2620   }
2621   if (!found)
2622     im->hbh_enabled = 0;
2623
2624   return (0);
2625 }
2626
2627 /* Global IP6 main. */
2628 ip6_main_t ip6_main;
2629
2630 static clib_error_t *
2631 ip6_lookup_init (vlib_main_t * vm)
2632 {
2633   ip6_main_t * im = &ip6_main;
2634   clib_error_t * error;
2635   uword i;
2636
2637   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2638     return error;
2639
2640   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2641     {
2642       u32 j, i0, i1;
2643
2644       i0 = i / 32;
2645       i1 = i % 32;
2646
2647       for (j = 0; j < i0; j++)
2648         im->fib_masks[i].as_u32[j] = ~0;
2649
2650       if (i1)
2651         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2652     }
2653
2654   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2655
2656   if (im->lookup_table_nbuckets == 0)
2657     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2658
2659   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2660
2661   if (im->lookup_table_size == 0)
2662     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2663
2664   BV(clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2665                         "ip6 FIB fwding table",
2666                         im->lookup_table_nbuckets,
2667                         im->lookup_table_size);
2668   BV(clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2669                         "ip6 FIB non-fwding table",
2670                         im->lookup_table_nbuckets,
2671                         im->lookup_table_size);
2672
2673   /* Create FIB with index 0 and table id of 0. */
2674   fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 0);
2675
2676   {
2677     pg_node_t * pn;
2678     pn = pg_get_node (ip6_lookup_node.index);
2679     pn->unformat_edit = unformat_pg_ip6_header;
2680   }
2681
2682   /* Unless explicitly configured, don't process HBH options */
2683   im->hbh_enabled = 0;
2684
2685   {
2686     icmp6_neighbor_solicitation_header_t p;
2687
2688     memset (&p, 0, sizeof (p));
2689
2690     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2691     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2692                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2693     p.ip.protocol = IP_PROTOCOL_ICMP6;
2694     p.ip.hop_limit = 255;
2695     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2696
2697     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2698
2699     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2700     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2701
2702     vlib_packet_template_init (vm,
2703                                &im->discover_neighbor_packet_template,
2704                                &p, sizeof (p),
2705                                /* alloc chunk size */ 8,
2706                                "ip6 neighbor discovery");
2707   }
2708
2709   return error;
2710 }
2711
2712 VLIB_INIT_FUNCTION (ip6_lookup_init);
2713
2714 static clib_error_t *
2715 add_del_ip6_interface_table (vlib_main_t * vm,
2716                              unformat_input_t * input,
2717                              vlib_cli_command_t * cmd)
2718 {
2719   vnet_main_t * vnm = vnet_get_main();
2720   clib_error_t * error = 0;
2721   u32 sw_if_index, table_id;
2722
2723   sw_if_index = ~0;
2724
2725   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2726     {
2727       error = clib_error_return (0, "unknown interface `%U'",
2728                                  format_unformat_error, input);
2729       goto done;
2730     }
2731
2732   if (unformat (input, "%d", &table_id))
2733     ;
2734   else
2735     {
2736       error = clib_error_return (0, "expected table id `%U'",
2737                                  format_unformat_error, input);
2738       goto done;
2739     }
2740
2741   {
2742     u32 fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6,
2743                                                       table_id);
2744
2745     vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
2746     ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
2747   }
2748
2749
2750  done:
2751   return error;
2752 }
2753
2754 /*?
2755  * Place the indicated interface into the supplied IPv6 FIB table (also known
2756  * as a VRF). If the FIB table does not exist, this command creates it. To
2757  * display the current IPv6 FIB table, use the command '<em>show ip6 fib</em>'.
2758  * FIB table will only be displayed if a route has been added to the table, or
2759  * an IP Address is assigned to an interface in the table (which adds a route
2760  * automatically).
2761  *
2762  * @note IP addresses added after setting the interface IP table end up in
2763  * the indicated FIB table. If the IP address is added prior to adding the
2764  * interface to the FIB table, it will NOT be part of the FIB table. Predictable
2765  * but potentially counter-intuitive results occur if you provision interface
2766  * addresses in multiple FIBs. Upon RX, packets will be processed in the last
2767  * IP table ID provisioned. It might be marginally useful to evade source RPF
2768  * drops to put an interface address into multiple FIBs.
2769  *
2770  * @cliexpar
2771  * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id):
2772  * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2}
2773  ?*/
2774 /* *INDENT-OFF* */
2775 VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) = {
2776   .path = "set interface ip6 table",
2777   .function = add_del_ip6_interface_table,
2778   .short_help = "set interface ip6 table <interface> <table-id>"
2779 };
2780 /* *INDENT-ON* */
2781
2782 void
2783 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
2784                                                   u8 *mac)
2785 {
2786   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2787   /* Invert the "u" bit */
2788   ip->as_u8 [8] = mac[0] ^ (1<<1);
2789   ip->as_u8 [9] = mac[1];
2790   ip->as_u8 [10] = mac[2];
2791   ip->as_u8 [11] = 0xFF;
2792   ip->as_u8 [12] = 0xFE;
2793   ip->as_u8 [13] = mac[3];
2794   ip->as_u8 [14] = mac[4];
2795   ip->as_u8 [15] = mac[5];
2796 }
2797
2798 void
2799 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
2800                                                   ip6_address_t *ip)
2801 {
2802   /* Invert the previously inverted "u" bit */
2803   mac[0] = ip->as_u8 [8] ^ (1<<1);
2804   mac[1] = ip->as_u8 [9];
2805   mac[2] = ip->as_u8 [10];
2806   mac[3] = ip->as_u8 [13];
2807   mac[4] = ip->as_u8 [14];
2808   mac[5] = ip->as_u8 [15];
2809 }
2810
2811 static clib_error_t *
2812 test_ip6_link_command_fn (vlib_main_t * vm,
2813                           unformat_input_t * input,
2814                           vlib_cli_command_t * cmd)
2815 {
2816   u8 mac[6];
2817   ip6_address_t _a, *a = &_a;
2818
2819   if (unformat (input, "%U", unformat_ethernet_address, mac))
2820     {
2821       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2822       vlib_cli_output (vm, "Link local address: %U",
2823                        format_ip6_address, a);
2824       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2825       vlib_cli_output (vm, "Original MAC address: %U",
2826                        format_ethernet_address, mac);
2827     }
2828
2829   return 0;
2830 }
2831
2832 /*?
2833  * This command converts the given MAC Address into an IPv6 link-local
2834  * address.
2835  *
2836  * @cliexpar
2837  * Example of how to create an IPv6 link-local address:
2838  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2839  * Link local address: fe80::14d9:e0ff:fe91:7986
2840  * Original MAC address: 16:d9:e0:91:79:86
2841  * @cliexend
2842 ?*/
2843 /* *INDENT-OFF* */
2844 VLIB_CLI_COMMAND (test_link_command, static) = {
2845   .path = "test ip6 link",
2846   .function = test_ip6_link_command_fn,
2847   .short_help = "test ip6 link <mac-address>",
2848 };
2849 /* *INDENT-ON* */
2850
2851 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2852 {
2853   ip6_main_t * im6 = &ip6_main;
2854   ip6_fib_t * fib;
2855   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
2856
2857   if (p == 0)
2858     return -1;
2859
2860   fib = ip6_fib_get (p[0]);
2861
2862   fib->flow_hash_config = flow_hash_config;
2863   return 1;
2864 }
2865
2866 static clib_error_t *
2867 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2868                               unformat_input_t * input,
2869                               vlib_cli_command_t * cmd)
2870 {
2871   int matched = 0;
2872   u32 table_id = 0;
2873   u32 flow_hash_config = 0;
2874   int rv;
2875
2876   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2877     if (unformat (input, "table %d", &table_id))
2878       matched = 1;
2879 #define _(a,v) \
2880     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2881     foreach_flow_hash_bit
2882 #undef _
2883     else break;
2884   }
2885
2886   if (matched == 0)
2887     return clib_error_return (0, "unknown input `%U'",
2888                               format_unformat_error, input);
2889
2890   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2891   switch (rv)
2892     {
2893     case 1:
2894       break;
2895
2896     case -1:
2897       return clib_error_return (0, "no such FIB table %d", table_id);
2898
2899     default:
2900       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2901       break;
2902     }
2903
2904   return 0;
2905 }
2906
2907 /*?
2908  * Configure the set of IPv6 fields used by the flow hash.
2909  *
2910  * @cliexpar
2911  * @parblock
2912  * Example of how to set the flow hash on a given table:
2913  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2914  *
2915  * Example of display the configured flow hash:
2916  * @cliexstart{show ip6 fib}
2917  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2918  * @::/0
2919  *   unicast-ip6-chain
2920  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2921  *     [0] [@0]: dpo-drop ip6
2922  * fe80::/10
2923  *   unicast-ip6-chain
2924  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2925  *     [0] [@2]: dpo-receive
2926  * ff02::1/128
2927  *   unicast-ip6-chain
2928  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2929  *     [0] [@2]: dpo-receive
2930  * ff02::2/128
2931  *   unicast-ip6-chain
2932  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2933  *     [0] [@2]: dpo-receive
2934  * ff02::16/128
2935  *   unicast-ip6-chain
2936  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2937  *     [0] [@2]: dpo-receive
2938  * ff02::1:ff00:0/104
2939  *   unicast-ip6-chain
2940  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
2941  *     [0] [@2]: dpo-receive
2942  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
2943  * @::/0
2944  *   unicast-ip6-chain
2945  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
2946  *     [0] [@0]: dpo-drop ip6
2947  * @::a:1:1:0:4/126
2948  *   unicast-ip6-chain
2949  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
2950  *     [0] [@4]: ipv6-glean: af_packet0
2951  * @::a:1:1:0:7/128
2952  *   unicast-ip6-chain
2953  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
2954  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
2955  * fe80::/10
2956  *   unicast-ip6-chain
2957  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
2958  *     [0] [@2]: dpo-receive
2959  * fe80::fe:3eff:fe3e:9222/128
2960  *   unicast-ip6-chain
2961  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
2962  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
2963  * ff02::1/128
2964  *   unicast-ip6-chain
2965  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
2966  *     [0] [@2]: dpo-receive
2967  * ff02::2/128
2968  *   unicast-ip6-chain
2969  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
2970  *     [0] [@2]: dpo-receive
2971  * ff02::16/128
2972  *   unicast-ip6-chain
2973  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
2974  *     [0] [@2]: dpo-receive
2975  * ff02::1:ff00:0/104
2976  *   unicast-ip6-chain
2977  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
2978  *     [0] [@2]: dpo-receive
2979  * @cliexend
2980  * @endparblock
2981 ?*/
2982 /* *INDENT-OFF* */
2983 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
2984     .path = "set ip6 flow-hash",
2985     .short_help =
2986     "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
2987     .function = set_ip6_flow_hash_command_fn,
2988 };
2989 /* *INDENT-ON* */
2990
2991 static clib_error_t *
2992 show_ip6_local_command_fn (vlib_main_t * vm,
2993                            unformat_input_t * input,
2994                            vlib_cli_command_t * cmd)
2995 {
2996   ip6_main_t * im = &ip6_main;
2997   ip_lookup_main_t * lm = &im->lookup_main;
2998   int i;
2999
3000   vlib_cli_output (vm, "Protocols handled by ip6_local");
3001   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
3002     {
3003       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3004         vlib_cli_output (vm, "%d", i);
3005     }
3006   return 0;
3007 }
3008
3009
3010
3011 /*?
3012  * Display the set of protocols handled by the local IPv6 stack.
3013  *
3014  * @cliexpar
3015  * Example of how to display local protocol table:
3016  * @cliexstart{show ip6 local}
3017  * Protocols handled by ip6_local
3018  * 17
3019  * 43
3020  * 58
3021  * 115
3022  * @cliexend
3023 ?*/
3024 /* *INDENT-OFF* */
3025 VLIB_CLI_COMMAND (show_ip6_local, static) = {
3026   .path = "show ip6 local",
3027   .function = show_ip6_local_command_fn,
3028   .short_help = "show ip6 local",
3029 };
3030 /* *INDENT-ON* */
3031
3032 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3033                                  u32 table_index)
3034 {
3035   vnet_main_t * vnm = vnet_get_main();
3036   vnet_interface_main_t * im = &vnm->interface_main;
3037   ip6_main_t * ipm = &ip6_main;
3038   ip_lookup_main_t * lm = &ipm->lookup_main;
3039   vnet_classify_main_t * cm = &vnet_classify_main;
3040   ip6_address_t *if_addr;
3041
3042   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3043     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3044
3045   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3046     return VNET_API_ERROR_NO_SUCH_ENTRY;
3047
3048   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3049   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
3050
3051   if_addr = ip6_interface_first_address (ipm, sw_if_index, NULL);
3052
3053   if (NULL != if_addr)
3054   {
3055       fib_prefix_t pfx = {
3056           .fp_len = 128,
3057           .fp_proto = FIB_PROTOCOL_IP6,
3058           .fp_addr.ip6 = *if_addr,
3059       };
3060       u32 fib_index;
3061
3062       fib_index = fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
3063                                                       sw_if_index);
3064
3065
3066       if (table_index != (u32) ~0)
3067       {
3068           dpo_id_t dpo = DPO_INVALID;
3069
3070           dpo_set(&dpo,
3071                   DPO_CLASSIFY,
3072                   DPO_PROTO_IP6,
3073                   classify_dpo_create(DPO_PROTO_IP6,
3074                                       table_index));
3075
3076           fib_table_entry_special_dpo_add(fib_index,
3077                                           &pfx,
3078                                           FIB_SOURCE_CLASSIFY,
3079                                           FIB_ENTRY_FLAG_NONE,
3080                                           &dpo);
3081           dpo_reset(&dpo);
3082       }
3083       else
3084       {
3085           fib_table_entry_special_remove(fib_index,
3086                                          &pfx,
3087                                          FIB_SOURCE_CLASSIFY);
3088       }
3089   }
3090
3091   return 0;
3092 }
3093
3094 static clib_error_t *
3095 set_ip6_classify_command_fn (vlib_main_t * vm,
3096                              unformat_input_t * input,
3097                              vlib_cli_command_t * cmd)
3098 {
3099   u32 table_index = ~0;
3100   int table_index_set = 0;
3101   u32 sw_if_index = ~0;
3102   int rv;
3103
3104   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3105     if (unformat (input, "table-index %d", &table_index))
3106       table_index_set = 1;
3107     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3108                        vnet_get_main(), &sw_if_index))
3109         ;
3110     else
3111         break;
3112   }
3113
3114   if (table_index_set == 0)
3115       return clib_error_return (0, "classify table-index must be specified");
3116
3117   if (sw_if_index == ~0)
3118     return clib_error_return (0, "interface / subif must be specified");
3119
3120   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3121
3122   switch (rv)
3123     {
3124     case 0:
3125       break;
3126
3127     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3128       return clib_error_return (0, "No such interface");
3129
3130     case VNET_API_ERROR_NO_SUCH_ENTRY:
3131       return clib_error_return (0, "No such classifier table");
3132     }
3133   return 0;
3134 }
3135
3136 /*?
3137  * Assign a classification table to an interface. The classification
3138  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3139  * commands. Once the table is create, use this command to filter packets
3140  * on an interface.
3141  *
3142  * @cliexpar
3143  * Example of how to assign a classification table to an interface:
3144  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3145 ?*/
3146 /* *INDENT-OFF* */
3147 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3148     .path = "set ip6 classify",
3149     .short_help =
3150     "set ip6 classify intfc <interface> table-index <classify-idx>",
3151     .function = set_ip6_classify_command_fn,
3152 };
3153 /* *INDENT-ON* */
3154
3155 static clib_error_t *
3156 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3157 {
3158   ip6_main_t * im = &ip6_main;
3159   uword heapsize = 0;
3160   u32 tmp;
3161   u32 nbuckets = 0;
3162
3163   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3164     if (unformat (input, "hash-buckets %d", &tmp))
3165       nbuckets = tmp;
3166     else if (unformat (input, "heap-size %dm", &tmp))
3167       heapsize = ((u64)tmp) << 20;
3168     else if (unformat (input, "heap-size %dM", &tmp))
3169       heapsize = ((u64)tmp) << 20;
3170     else if (unformat (input, "heap-size %dg", &tmp))
3171       heapsize = ((u64)tmp) << 30;
3172     else if (unformat (input, "heap-size %dG", &tmp))
3173       heapsize = ((u64)tmp) << 30;
3174     else
3175       return clib_error_return (0, "unknown input '%U'",
3176                                 format_unformat_error, input);
3177   }
3178
3179   im->lookup_table_nbuckets = nbuckets;
3180   im->lookup_table_size = heapsize;
3181
3182   return 0;
3183 }
3184
3185 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");