vnet/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
  43 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  44 #include <vppinfra/cache.h>
  45 #include <vnet/fib/fib_table.h>
  46 #include <vnet/fib/ip6_fib.h>
  47 #include <vnet/dpo/load_balance.h>
  48 #include <vnet/dpo/classify_dpo.h>
  49
  50 #include <vppinfra/bihash_template.c>
  51
  52 void
  53 ip6_forward_next_trace (vlib_main_t * vm,
  54                         vlib_node_runtime_t * node,
  55                         vlib_frame_t * frame,
  56                         vlib_rx_or_tx_t which_adj_index);
  57
  58 always_inline uword
  59 ip6_lookup_inline (vlib_main_t * vm,
  60                    vlib_node_runtime_t * node,
  61                    vlib_frame_t * frame)
  62 {
  63   ip6_main_t * im = &ip6_main;
  64   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters;
  65   u32 n_left_from, n_left_to_next, * from, * to_next;
  66   ip_lookup_next_t next;
  67   u32 cpu_index = os_get_cpu_number();
  68
  69   from = vlib_frame_vector_args (frame);
  70   n_left_from = frame->n_vectors;
  71   next = node->cached_next_index;
  72
  73   while (n_left_from > 0)
  74     {
  75       vlib_get_next_frame (vm, node, next,
  76                            to_next, n_left_to_next);
  77
  78       while (n_left_from >= 4 && n_left_to_next >= 2)
  79         {
  80           vlib_buffer_t * p0, * p1;
  81           u32 pi0, pi1, lbi0, lbi1, wrong_next;
  82           ip_lookup_next_t next0, next1;
  83           ip6_header_t * ip0, * ip1;
  84           ip6_address_t * dst_addr0, * dst_addr1;
  85           u32 fib_index0, fib_index1;
  86           u32 flow_hash_config0, flow_hash_config1;
  87           const dpo_id_t *dpo0, *dpo1;
  88           const load_balance_t *lb0, *lb1;
  89
  90           /* Prefetch next iteration. */
  91           {
  92             vlib_buffer_t * p2, * p3;
  93
  94             p2 = vlib_get_buffer (vm, from[2]);
  95             p3 = vlib_get_buffer (vm, from[3]);
  96
  97             vlib_prefetch_buffer_header (p2, LOAD);
  98             vlib_prefetch_buffer_header (p3, LOAD);
  99             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
 100             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
 101           }
 102
 103           pi0 = to_next[0] = from[0];
 104           pi1 = to_next[1] = from[1];
 105
 106           p0 = vlib_get_buffer (vm, pi0);
 107           p1 = vlib_get_buffer (vm, pi1);
 108
 109           ip0 = vlib_buffer_get_current (p0);
 110           ip1 = vlib_buffer_get_current (p1);
 111
 112           dst_addr0 = &ip0->dst_address;
 113           dst_addr1 = &ip1->dst_address;
 114
 115           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 116           fib_index1 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p1)->sw_if_index[VLIB_RX]);
 117
 118           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 119             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 120           fib_index1 = (vnet_buffer(p1)->sw_if_index[VLIB_TX] == (u32)~0) ?
 121             fib_index1 : vnet_buffer(p1)->sw_if_index[VLIB_TX];
 122
 123           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 124           lbi1 = ip6_fib_table_fwding_lookup (im, fib_index1, dst_addr1);
 125
 126           lb0 = load_balance_get (lbi0);
 127           lb1 = load_balance_get (lbi1);
 128
 129           vnet_buffer (p0)->ip.flow_hash =
 130             vnet_buffer(p1)->ip.flow_hash = 0;
 131
 132           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 133             {
 134               flow_hash_config0 = lb0->lb_hash_config;
 135               vnet_buffer (p0)->ip.flow_hash =
 136                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 137             }
 138           if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
 139             {
 140               flow_hash_config1 = lb1->lb_hash_config;
 141               vnet_buffer (p1)->ip.flow_hash =
 142                 ip6_compute_flow_hash (ip1, flow_hash_config1);
 143             }
 144
 145           ASSERT (lb0->lb_n_buckets > 0);
 146           ASSERT (lb1->lb_n_buckets > 0);
 147           ASSERT (is_pow2 (lb0->lb_n_buckets));
 148           ASSERT (is_pow2 (lb1->lb_n_buckets));
 149           dpo0 = load_balance_get_bucket_i(lb0,
 150                                            (vnet_buffer (p0)->ip.flow_hash &
 151                                             lb0->lb_n_buckets_minus_1));
 152           dpo1 = load_balance_get_bucket_i(lb1,
 153                                            (vnet_buffer (p1)->ip.flow_hash &
 154                                             lb1->lb_n_buckets_minus_1));
 155
 156           next0 = dpo0->dpoi_next_node;
 157           next1 = dpo1->dpoi_next_node;
 158
 159           /* Only process the HBH Option Header if explicitly configured to do so */
 160           next0 = ((ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) &&
 161                    im->hbh_enabled) ?
 162             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP :
 163             next0;
 164           next1 = ((ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) &&
 165                    im->hbh_enabled) ?
 166             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP :
 167             next1;
 168
 169           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 170           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 171
 172           vlib_increment_combined_counter
 173               (cm, cpu_index, lbi0, 1,
 174                vlib_buffer_length_in_chain (vm, p0));
 175           vlib_increment_combined_counter
 176               (cm, cpu_index, lbi1, 1,
 177                vlib_buffer_length_in_chain (vm, p1));
 178
 179           from += 2;
 180           to_next += 2;
 181           n_left_to_next -= 2;
 182           n_left_from -= 2;
 183
 184           wrong_next = (next0 != next) + 2*(next1 != next);
 185           if (PREDICT_FALSE (wrong_next != 0))
 186             {
 187               switch (wrong_next)
 188                 {
 189                 case 1:
 190                   /* A B A */
 191                   to_next[-2] = pi1;
 192                   to_next -= 1;
 193                   n_left_to_next += 1;
 194                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 195                   break;
 196
 197                 case 2:
 198                   /* A A B */
 199                   to_next -= 1;
 200                   n_left_to_next += 1;
 201                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 202                   break;
 203
 204                 case 3:
 205                   /* A B C */
 206                   to_next -= 2;
 207                   n_left_to_next += 2;
 208                   vlib_set_next_frame_buffer (vm, node, next0, pi0);
 209                   vlib_set_next_frame_buffer (vm, node, next1, pi1);
 210                   if (next0 == next1)
 211                     {
 212                       /* A B B */
 213                       vlib_put_next_frame (vm, node, next, n_left_to_next);
 214                       next = next1;
 215                       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 216                     }
 217                 }
 218             }
 219         }
 220
 221       while (n_left_from > 0 && n_left_to_next > 0)
 222         {
 223           vlib_buffer_t * p0;
 224           ip6_header_t * ip0;
 225           u32 pi0, lbi0;
 226           ip_lookup_next_t next0;
 227           load_balance_t * lb0;
 228           ip6_address_t * dst_addr0;
 229           u32 fib_index0, flow_hash_config0;
 230           const dpo_id_t *dpo0;
 231
 232           pi0 = from[0];
 233           to_next[0] = pi0;
 234
 235           p0 = vlib_get_buffer (vm, pi0);
 236
 237           ip0 = vlib_buffer_get_current (p0);
 238
 239           dst_addr0 = &ip0->dst_address;
 240
 241           fib_index0 = vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (p0)->sw_if_index[VLIB_RX]);
 242           fib_index0 = (vnet_buffer(p0)->sw_if_index[VLIB_TX] == (u32)~0) ?
 243             fib_index0 : vnet_buffer(p0)->sw_if_index[VLIB_TX];
 244
 245           flow_hash_config0 =
 246               ip6_fib_get (fib_index0)->flow_hash_config;
 247
 248           lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 249
 250           lb0 = load_balance_get (lbi0);
 251
 252           vnet_buffer (p0)->ip.flow_hash = 0;
 253
 254           if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
 255             {
 256               flow_hash_config0 = lb0->lb_hash_config;
 257               vnet_buffer (p0)->ip.flow_hash =
 258                 ip6_compute_flow_hash (ip0, flow_hash_config0);
 259             }
 260
 261           ASSERT (lb0->lb_n_buckets > 0);
 262           ASSERT (is_pow2 (lb0->lb_n_buckets));
 263           dpo0 = load_balance_get_bucket_i(lb0,
 264                                            (vnet_buffer (p0)->ip.flow_hash &
 265                                             lb0->lb_n_buckets_minus_1));
 266           next0 = dpo0->dpoi_next_node;
 267           /* Only process the HBH Option Header if explicitly configured to do so */
 268           next0 = ((ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) &&
 269                    im->hbh_enabled) ?
 270             (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP :
 271             next0;
 272
 273           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 274
 275           vlib_increment_combined_counter
 276               (cm, cpu_index, lbi0, 1,
 277                vlib_buffer_length_in_chain (vm, p0));
 278
 279           from += 1;
 280           to_next += 1;
 281           n_left_to_next -= 1;
 282           n_left_from -= 1;
 283
 284           if (PREDICT_FALSE (next0 != next))
 285             {
 286               n_left_to_next += 1;
 287               vlib_put_next_frame (vm, node, next, n_left_to_next);
 288               next = next0;
 289               vlib_get_next_frame (vm, node, next,
 290                                    to_next, n_left_to_next);
 291               to_next[0] = pi0;
 292               to_next += 1;
 293               n_left_to_next -= 1;
 294             }
 295         }
 296
 297       vlib_put_next_frame (vm, node, next, n_left_to_next);
 298     }
 299
 300   if (node->flags & VLIB_NODE_FLAG_TRACE)
 301     ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 302
 303   return frame->n_vectors;
 304 }
 305
 306 static void
 307 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 308                           ip6_main_t * im, u32 fib_index,
 309                           ip_interface_address_t * a)
 310 {
 311   ip_lookup_main_t * lm = &im->lookup_main;
 312   ip6_address_t * address = ip_interface_address_get_address (lm, a);
 313   fib_prefix_t pfx = {
 314       .fp_len = a->address_length,
 315       .fp_proto = FIB_PROTOCOL_IP6,
 316       .fp_addr.ip6 = *address,
 317   };
 318
 319   a->neighbor_probe_adj_index = ~0;
 320   if (a->address_length < 128)
 321   {
 322       fib_node_index_t fei;
 323
 324       fei = fib_table_entry_update_one_path(fib_index,
 325                                             &pfx,
 326                                             FIB_SOURCE_INTERFACE,
 327                                             (FIB_ENTRY_FLAG_CONNECTED |
 328                                              FIB_ENTRY_FLAG_ATTACHED),
 329                                             FIB_PROTOCOL_IP6,
 330                                             NULL, /* No next-hop address */
 331                                             sw_if_index,
 332                                             ~0, // invalid FIB index
 333                                             1,
 334                                             MPLS_LABEL_INVALID,
 335                                             FIB_ROUTE_PATH_FLAG_NONE);
 336       a->neighbor_probe_adj_index = fib_entry_get_adj(fei);
 337   }
 338
 339   pfx.fp_len = 128;
 340   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
 341   {
 342       u32 classify_table_index =
 343           lm->classify_table_index_by_sw_if_index [sw_if_index];
 344       if (classify_table_index != (u32) ~0)
 345       {
 346           dpo_id_t dpo = DPO_NULL;
 347
 348           dpo_set(&dpo,
 349                   DPO_CLASSIFY,
 350                   DPO_PROTO_IP4,
 351                   classify_dpo_create(FIB_PROTOCOL_IP6,
 352                                       classify_table_index));
 353
 354           fib_table_entry_special_dpo_add(fib_index,
 355                                           &pfx,
 356                                           FIB_SOURCE_CLASSIFY,
 357                                           FIB_ENTRY_FLAG_NONE,
 358                                           &dpo);
 359           dpo_reset(&dpo);
 360       }
 361   }
 362
 363   fib_table_entry_update_one_path(fib_index,
 364                                   &pfx,
 365                                   FIB_SOURCE_INTERFACE,
 366                                   (FIB_ENTRY_FLAG_CONNECTED |
 367                                    FIB_ENTRY_FLAG_LOCAL),
 368                                   FIB_PROTOCOL_IP6,
 369                                   &pfx.fp_addr,
 370                                   sw_if_index,
 371                                   ~0, // invalid FIB index
 372                                   1,
 373                                   MPLS_LABEL_INVALID,
 374                                   FIB_ROUTE_PATH_FLAG_NONE);
 375 }
 376
 377 static void
 378 ip6_del_interface_routes (ip6_main_t * im,
 379                           u32 fib_index,
 380                           ip6_address_t * address,
 381                           u32 address_length)
 382 {
 383     fib_prefix_t pfx = {
 384         .fp_len = address_length,
 385         .fp_proto = FIB_PROTOCOL_IP6,
 386         .fp_addr.ip6 = *address,
 387     };
 388
 389     if (pfx.fp_len < 128)
 390     {
 391         fib_table_entry_delete(fib_index,
 392                                &pfx,
 393                                FIB_SOURCE_INTERFACE);
 394
 395     }
 396
 397     pfx.fp_len = 128;
 398     fib_table_entry_delete(fib_index,
 399                            &pfx,
 400                            FIB_SOURCE_INTERFACE);
 401 }
 402
 403 void
 404 ip6_sw_interface_enable_disable (u32 sw_if_index,
 405                                  u32 is_enable)
 406 {
 407   vlib_main_t * vm = vlib_get_main();
 408   ip6_main_t * im = &ip6_main;
 409   ip_lookup_main_t * lm = &im->lookup_main;
 410   u32 ci, cast;
 411   u32 lookup_feature_index;
 412
 413   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 414
 415   /*
 416    * enable/disable only on the 1<->0 transition
 417    */
 418   if (is_enable)
 419     {
 420       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 421         return;
 422     }
 423   else
 424     {
 425       ASSERT(im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
 426       if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 427         return;
 428     }
 429
 430   for (cast = 0; cast <= VNET_IP_RX_MULTICAST_FEAT; cast++)
 431     {
 432       ip_config_main_t * cm = &lm->feature_config_mains[cast];
 433       vnet_config_main_t * vcm = &cm->config_main;
 434
 435       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
 436       ci = cm->config_index_by_sw_if_index[sw_if_index];
 437
 438       if (cast == VNET_IP_RX_UNICAST_FEAT)
 439         lookup_feature_index = im->ip6_unicast_rx_feature_lookup;
 440       else
 441         lookup_feature_index = im->ip6_multicast_rx_feature_lookup;
 442
 443       if (is_enable)
 444         ci = vnet_config_add_feature (vm, vcm,
 445                                       ci,
 446                                       lookup_feature_index,
 447                                       /* config data */ 0,
 448                                       /* # bytes of config data */ 0);
 449       else
 450         ci = vnet_config_del_feature (vm, vcm,
 451                                       ci,
 452                                       lookup_feature_index,
 453                                       /* config data */ 0,
 454                                       /* # bytes of config data */ 0);
 455
 456       cm->config_index_by_sw_if_index[sw_if_index] = ci;
 457     }
 458 }
 459
 460 /* get first interface address */
 461 ip6_address_t *
 462 ip6_interface_first_address (ip6_main_t * im,
 463                              u32 sw_if_index,
 464                              ip_interface_address_t ** result_ia)
 465 {
 466   ip_lookup_main_t * lm = &im->lookup_main;
 467   ip_interface_address_t * ia = 0;
 468   ip6_address_t * result = 0;
 469
 470   foreach_ip_interface_address (lm, ia, sw_if_index,
 471                                 1 /* honor unnumbered */,
 472   ({
 473     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 474     result = a;
 475     break;
 476   }));
 477   if (result_ia)
 478     *result_ia = result ? ia : 0;
 479   return result;
 480 }
 481
 482 clib_error_t *
 483 ip6_add_del_interface_address (vlib_main_t * vm,
 484                                u32 sw_if_index,
 485                                ip6_address_t * address,
 486                                u32 address_length,
 487                                u32 is_del)
 488 {
 489   vnet_main_t * vnm = vnet_get_main();
 490   ip6_main_t * im = &ip6_main;
 491   ip_lookup_main_t * lm = &im->lookup_main;
 492   clib_error_t * error;
 493   u32 if_address_index;
 494   ip6_address_fib_t ip6_af, * addr_fib = 0;
 495
 496   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 497   ip6_addr_fib_init (&ip6_af, address,
 498                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 499   vec_add1 (addr_fib, ip6_af);
 500
 501   {
 502     uword elts_before = pool_elts (lm->if_address_pool);
 503
 504     error = ip_interface_address_add_del
 505       (lm,
 506        sw_if_index,
 507        addr_fib,
 508        address_length,
 509        is_del,
 510        &if_address_index);
 511     if (error)
 512       goto done;
 513
 514     /* Pool did not grow: add duplicate address. */
 515     if (elts_before == pool_elts (lm->if_address_pool))
 516       goto done;
 517   }
 518
 519   if (is_del)
 520       ip6_del_interface_routes (im, ip6_af.fib_index, address,
 521                                 address_length);
 522   else
 523       ip6_add_interface_routes (vnm, sw_if_index,
 524                                 im, ip6_af.fib_index,
 525                                 pool_elt_at_index (lm->if_address_pool, if_address_index));
 526
 527   {
 528     ip6_add_del_interface_address_callback_t * cb;
 529     vec_foreach (cb, im->add_del_interface_address_callbacks)
 530       cb->function (im, cb->function_opaque, sw_if_index,
 531                     address, address_length,
 532                     if_address_index,
 533                     is_del);
 534   }
 535
 536  done:
 537   vec_free (addr_fib);
 538   return error;
 539 }
 540
 541 clib_error_t *
 542 ip6_sw_interface_admin_up_down (vnet_main_t * vnm,
 543                                 u32 sw_if_index,
 544                                 u32 flags)
 545 {
 546   ip6_main_t * im = &ip6_main;
 547   ip_interface_address_t * ia;
 548   ip6_address_t * a;
 549   u32 is_admin_up, fib_index;
 550
 551   /* Fill in lookup tables with default table (0). */
 552   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 553
 554   vec_validate_init_empty (im->lookup_main.if_address_pool_index_by_sw_if_index, sw_if_index, ~0);
 555
 556   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 557
 558   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 559
 560   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 561                                 0 /* honor unnumbered */,
 562   ({
 563     a = ip_interface_address_get_address (&im->lookup_main, ia);
 564     if (is_admin_up)
 565       ip6_add_interface_routes (vnm, sw_if_index,
 566                                 im, fib_index,
 567                                 ia);
 568     else
 569       ip6_del_interface_routes (im, fib_index,
 570                                 a, ia->address_length);
 571   }));
 572
 573   return 0;
 574 }
 575
 576 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 577
 578 /* Built-in ip6 unicast rx feature path definition */
 579 VNET_IP6_UNICAST_FEATURE_INIT (ip6_inacl, static) = {
 580   .node_name = "ip6-inacl",
 581   .runs_before = ORDER_CONSTRAINTS {"ip6-policer-classify", 0},
 582   .feature_index = &ip6_main.ip6_unicast_rx_feature_check_access,
 583 };
 584
 585 VNET_IP6_UNICAST_FEATURE_INIT (ip6_policer_classify, static) = {
 586   .node_name = "ip6-policer-classify",
 587   .runs_before = ORDER_CONSTRAINTS {"ipsec-input-ip6", 0},
 588   .feature_index = &ip6_main.ip6_unicast_rx_feature_policer_classify,
 589 };
 590
 591 VNET_IP6_UNICAST_FEATURE_INIT (ip6_ipsec, static) = {
 592   .node_name = "ipsec-input-ip6",
 593   .runs_before = ORDER_CONSTRAINTS {"l2tp-decap", 0},
 594   .feature_index = &ip6_main.ip6_unicast_rx_feature_ipsec,
 595 };
 596
 597 VNET_IP6_UNICAST_FEATURE_INIT (ip6_l2tp, static) = {
 598   .node_name = "l2tp-decap",
 599   .runs_before = ORDER_CONSTRAINTS {"vpath-input-ip6", 0},
 600   .feature_index = &ip6_main.ip6_unicast_rx_feature_l2tp_decap,
 601 };
 602
 603 VNET_IP6_UNICAST_FEATURE_INIT (ip6_vpath, static) = {
 604   .node_name = "vpath-input-ip6",
 605   .runs_before = ORDER_CONSTRAINTS {"ip6-lookup", 0},
 606   .feature_index = &ip6_main.ip6_unicast_rx_feature_vpath,
 607 };
 608
 609 VNET_IP6_UNICAST_FEATURE_INIT (ip6_lookup, static) = {
 610   .node_name = "ip6-lookup",
 611   .runs_before = ORDER_CONSTRAINTS {"ip6-drop", 0},
 612   .feature_index = &ip6_main.ip6_unicast_rx_feature_lookup,
 613 };
 614
 615 VNET_IP6_UNICAST_FEATURE_INIT (ip6_drop, static) = {
 616   .node_name = "ip6-drop",
 617   .runs_before = 0,  /*last feature*/
 618   .feature_index = &ip6_main.ip6_unicast_rx_feature_drop,
 619 };
 620
 621 /* Built-in ip6 multicast rx feature path definition (none now) */
 622 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_vpath_mc, static) = {
 623   .node_name = "vpath-input-ip6",
 624   .runs_before = ORDER_CONSTRAINTS {"ip6-lookup", 0},
 625   .feature_index = &ip6_main.ip6_multicast_rx_feature_vpath,
 626 };
 627
 628 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_lookup, static) = {
 629   .node_name = "ip6-lookup",
 630   .runs_before = ORDER_CONSTRAINTS {"ip6-drop", 0},
 631   .feature_index = &ip6_main.ip6_multicast_rx_feature_lookup,
 632 };
 633
 634 VNET_IP6_MULTICAST_FEATURE_INIT (ip6_drop_mc, static) = {
 635   .node_name = "ip6-drop",
 636   .runs_before = 0, /* last feature */
 637   .feature_index = &ip6_main.ip6_multicast_rx_feature_drop,
 638 };
 639
 640 static char * rx_feature_start_nodes[] =
 641   {"ip6-input"};
 642
 643 static char * tx_feature_start_nodes[] =
 644 {
 645   "ip6-rewrite",
 646   "ip6-midchain",
 647 };
 648
 649 /* Built-in ip4 tx feature path definition */
 650 VNET_IP6_TX_FEATURE_INIT (interface_output, static) = {
 651   .node_name = "interface-output",
 652   .runs_before = 0, /* not before any other features */
 653   .feature_index = &ip6_main.ip6_tx_feature_interface_output,
 654 };
 655
 656 static clib_error_t *
 657 ip6_feature_init (vlib_main_t * vm, ip6_main_t * im)
 658 {
 659   ip_lookup_main_t * lm = &im->lookup_main;
 660   clib_error_t * error;
 661   vnet_cast_t cast;
 662   ip_config_main_t * cm;
 663   vnet_config_main_t * vcm;
 664   char **feature_start_nodes;
 665   int feature_start_len;
 666
 667   for (cast = 0; cast < VNET_N_IP_FEAT; cast++)
 668     {
 669       cm = &lm->feature_config_mains[cast];
 670       vcm = &cm->config_main;
 671
 672       if (cast < VNET_IP_TX_FEAT)
 673         {
 674           feature_start_nodes = rx_feature_start_nodes;
 675           feature_start_len = ARRAY_LEN(rx_feature_start_nodes);
 676         }
 677       else
 678         {
 679           feature_start_nodes = tx_feature_start_nodes;
 680           feature_start_len = ARRAY_LEN(tx_feature_start_nodes);
 681         }
 682
 683       if ((error = ip_feature_init_cast (vm, cm, vcm,
 684                                          feature_start_nodes,
 685                                          feature_start_len,
 686                                          im->next_feature[cast],
 687                                          &im->feature_nodes[cast])))
 688         return error;
 689     }
 690   return 0;
 691 }
 692
 693 clib_error_t *
 694 ip6_sw_interface_add_del (vnet_main_t * vnm,
 695                           u32 sw_if_index,
 696                           u32 is_add)
 697 {
 698   vlib_main_t * vm = vnm->vlib_main;
 699   ip6_main_t * im = &ip6_main;
 700   ip_lookup_main_t * lm = &im->lookup_main;
 701   u32 ci, cast;
 702   u32 feature_index;
 703
 704   for (cast = 0; cast < VNET_N_IP_FEAT; cast++)
 705     {
 706       ip_config_main_t * cm = &lm->feature_config_mains[cast];
 707       vnet_config_main_t * vcm = &cm->config_main;
 708
 709       vec_validate_init_empty (cm->config_index_by_sw_if_index, sw_if_index, ~0);
 710       ci = cm->config_index_by_sw_if_index[sw_if_index];
 711
 712       if (cast == VNET_IP_RX_UNICAST_FEAT)
 713         feature_index = im->ip6_unicast_rx_feature_drop;
 714       else if (cast == VNET_IP_RX_MULTICAST_FEAT)
 715         feature_index = im->ip6_multicast_rx_feature_drop;
 716       else
 717         feature_index = im->ip6_tx_feature_interface_output;
 718
 719       if (is_add)
 720         ci = vnet_config_add_feature (vm, vcm,
 721                                       ci,
 722                                       feature_index,
 723                                       /* config data */ 0,
 724                                       /* # bytes of config data */ 0);
 725       else
 726         {
 727           ci = vnet_config_del_feature (vm, vcm, ci,
 728                                         feature_index,
 729                                         /* config data */ 0,
 730                                         /* # bytes of config data */ 0);
 731           if (vec_len(im->ip_enabled_by_sw_if_index) > sw_if_index)
 732               im->ip_enabled_by_sw_if_index[sw_if_index] = 0;
 733         }
 734       cm->config_index_by_sw_if_index[sw_if_index] = ci;
 735       /*
 736        * note: do not update the tx feature count here.
 737        */
 738     }
 739   return /* no error */ 0;
 740 }
 741
 742 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 743
 744 static uword
 745 ip6_lookup (vlib_main_t * vm,
 746             vlib_node_runtime_t * node,
 747             vlib_frame_t * frame)
 748 {
 749   return ip6_lookup_inline (vm, node, frame);
 750 }
 751
 752 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args);
 753
 754 VLIB_REGISTER_NODE (ip6_lookup_node) = {
 755   .function = ip6_lookup,
 756   .name = "ip6-lookup",
 757   .vector_size = sizeof (u32),
 758
 759   .format_trace = format_ip6_lookup_trace,
 760
 761   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 762   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 763 };
 764
 765 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup)
 766
 767 always_inline uword
 768 ip6_load_balance (vlib_main_t * vm,
 769                   vlib_node_runtime_t * node,
 770                   vlib_frame_t * frame)
 771 {
 772   vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters;
 773   u32 n_left_from, n_left_to_next, * from, * to_next;
 774   ip_lookup_next_t next;
 775   u32 cpu_index = os_get_cpu_number();
 776
 777   from = vlib_frame_vector_args (frame);
 778   n_left_from = frame->n_vectors;
 779   next = node->cached_next_index;
 780
 781   if (node->flags & VLIB_NODE_FLAG_TRACE)
 782       ip6_forward_next_trace(vm, node, frame, VLIB_TX);
 783
 784   while (n_left_from > 0)
 785     {
 786       vlib_get_next_frame (vm, node, next,
 787                            to_next, n_left_to_next);
 788
 789
 790       while (n_left_from > 0 && n_left_to_next > 0)
 791         {
 792           ip_lookup_next_t next0;
 793           const load_balance_t *lb0;
 794           vlib_buffer_t * p0;
 795           u32 pi0, lbi0, hc0;
 796           const ip6_header_t *ip0;
 797           const dpo_id_t *dpo0;
 798
 799           pi0 = from[0];
 800           to_next[0] = pi0;
 801
 802           p0 = vlib_get_buffer (vm, pi0);
 803
 804           ip0 = vlib_buffer_get_current (p0);
 805           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 806
 807           lb0 = load_balance_get(lbi0);
 808           hc0 = lb0->lb_hash_config;
 809           vnet_buffer(p0)->ip.flow_hash = ip6_compute_flow_hash(ip0, hc0);
 810
 811           dpo0 = load_balance_get_bucket_i(lb0,
 812                                            vnet_buffer(p0)->ip.flow_hash &
 813                                            (lb0->lb_n_buckets - 1));
 814
 815           next0 = dpo0->dpoi_next_node;
 816           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 817
 818           vlib_increment_combined_counter
 819               (cm, cpu_index, lbi0, 1,
 820                vlib_buffer_length_in_chain (vm, p0));
 821
 822           from += 1;
 823           to_next += 1;
 824           n_left_to_next -= 1;
 825           n_left_from -= 1;
 826
 827           if (PREDICT_FALSE (next0 != next))
 828             {
 829               n_left_to_next += 1;
 830               vlib_put_next_frame (vm, node, next, n_left_to_next);
 831               next = next0;
 832               vlib_get_next_frame (vm, node, next,
 833                                    to_next, n_left_to_next);
 834               to_next[0] = pi0;
 835               to_next += 1;
 836               n_left_to_next -= 1;
 837             }
 838         }
 839
 840       vlib_put_next_frame (vm, node, next, n_left_to_next);
 841     }
 842
 843   return frame->n_vectors;
 844 }
 845
 846 VLIB_REGISTER_NODE (ip6_load_balance_node) = {
 847   .function = ip6_load_balance,
 848   .name = "ip6-load-balance",
 849   .vector_size = sizeof (u32),
 850   .sibling_of = "ip6-lookup",
 851   .format_trace = format_ip6_lookup_trace,
 852   .n_next_nodes = 0,
 853 };
 854
 855 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance)
 856
 857 typedef struct {
 858   /* Adjacency taken. */
 859   u32 adj_index;
 860   u32 flow_hash;
 861   u32 fib_index;
 862
 863   /* Packet data, possibly *after* rewrite. */
 864   u8 packet_data[128 - 1*sizeof(u32)];
 865 } ip6_forward_next_trace_t;
 866
 867 static u8 * format_ip6_forward_next_trace (u8 * s, va_list * args)
 868 {
 869   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 870   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 871   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 872   uword indent = format_get_indent (s);
 873
 874   s = format(s, "%U%U",
 875              format_white_space, indent,
 876              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 877   return s;
 878 }
 879
 880 static u8 * format_ip6_lookup_trace (u8 * s, va_list * args)
 881 {
 882   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 883   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 884   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 885   uword indent = format_get_indent (s);
 886
 887   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 888               t->fib_index, t->adj_index, t->flow_hash);
 889   s = format(s, "\n%U%U",
 890              format_white_space, indent,
 891              format_ip6_header, t->packet_data, sizeof (t->packet_data));
 892   return s;
 893 }
 894
 895
 896 static u8 * format_ip6_rewrite_trace (u8 * s, va_list * args)
 897 {
 898   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 899   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 900   ip6_forward_next_trace_t * t = va_arg (*args, ip6_forward_next_trace_t *);
 901   vnet_main_t * vnm = vnet_get_main();
 902   uword indent = format_get_indent (s);
 903
 904   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 905               t->fib_index, t->adj_index, format_ip_adjacency,
 906               vnm, t->adj_index, FORMAT_IP_ADJACENCY_NONE,
 907               t->flow_hash);
 908   s = format (s, "\n%U%U",
 909               format_white_space, indent,
 910               format_ip_adjacency_packet_data,
 911               vnm, t->adj_index,
 912               t->packet_data, sizeof (t->packet_data));
 913   return s;
 914 }
 915
 916 /* Common trace function for all ip6-forward next nodes. */
 917 void
 918 ip6_forward_next_trace (vlib_main_t * vm,
 919                         vlib_node_runtime_t * node,
 920                         vlib_frame_t * frame,
 921                         vlib_rx_or_tx_t which_adj_index)
 922 {
 923   u32 * from, n_left;
 924   ip6_main_t * im = &ip6_main;
 925
 926   n_left = frame->n_vectors;
 927   from = vlib_frame_vector_args (frame);
 928
 929   while (n_left >= 4)
 930     {
 931       u32 bi0, bi1;
 932       vlib_buffer_t * b0, * b1;
 933       ip6_forward_next_trace_t * t0, * t1;
 934
 935       /* Prefetch next iteration. */
 936       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 937       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 938
 939       bi0 = from[0];
 940       bi1 = from[1];
 941
 942       b0 = vlib_get_buffer (vm, bi0);
 943       b1 = vlib_get_buffer (vm, bi1);
 944
 945       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 946         {
 947           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 948           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 949           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 950           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
 951               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
 952               vec_elt (im->fib_index_by_sw_if_index,
 953                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 954
 955           clib_memcpy (t0->packet_data,
 956                   vlib_buffer_get_current (b0),
 957                   sizeof (t0->packet_data));
 958         }
 959       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 960         {
 961           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 962           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 963           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 964           t1->fib_index = (vnet_buffer(b1)->sw_if_index[VLIB_TX] != (u32)~0) ?
 965               vnet_buffer(b1)->sw_if_index[VLIB_TX] :
 966               vec_elt (im->fib_index_by_sw_if_index,
 967                        vnet_buffer(b1)->sw_if_index[VLIB_RX]);
 968
 969           clib_memcpy (t1->packet_data,
 970                   vlib_buffer_get_current (b1),
 971                   sizeof (t1->packet_data));
 972         }
 973       from += 2;
 974       n_left -= 2;
 975     }
 976
 977   while (n_left >= 1)
 978     {
 979       u32 bi0;
 980       vlib_buffer_t * b0;
 981       ip6_forward_next_trace_t * t0;
 982
 983       bi0 = from[0];
 984
 985       b0 = vlib_get_buffer (vm, bi0);
 986
 987       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 988         {
 989           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 990           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 991           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 992           t0->fib_index = (vnet_buffer(b0)->sw_if_index[VLIB_TX] != (u32)~0) ?
 993               vnet_buffer(b0)->sw_if_index[VLIB_TX] :
 994               vec_elt (im->fib_index_by_sw_if_index,
 995                        vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 996
 997           clib_memcpy (t0->packet_data,
 998                   vlib_buffer_get_current (b0),
 999                   sizeof (t0->packet_data));
1000         }
1001       from += 1;
1002       n_left -= 1;
1003     }
1004 }
1005
1006 static uword
1007 ip6_drop_or_punt (vlib_main_t * vm,
1008                   vlib_node_runtime_t * node,
1009                   vlib_frame_t * frame,
1010                   ip6_error_t error_code)
1011 {
1012   u32 * buffers = vlib_frame_vector_args (frame);
1013   uword n_packets = frame->n_vectors;
1014
1015   vlib_error_drop_buffers (vm, node,
1016                            buffers,
1017                            /* stride */ 1,
1018                            n_packets,
1019                            /* next */ 0,
1020                            ip6_input_node.index,
1021                            error_code);
1022
1023   if (node->flags & VLIB_NODE_FLAG_TRACE)
1024     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1025
1026   return n_packets;
1027 }
1028
1029 static uword
1030 ip6_drop (vlib_main_t * vm,
1031           vlib_node_runtime_t * node,
1032           vlib_frame_t * frame)
1033 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_DROP); }
1034
1035 static uword
1036 ip6_punt (vlib_main_t * vm,
1037           vlib_node_runtime_t * node,
1038           vlib_frame_t * frame)
1039 { return ip6_drop_or_punt (vm, node, frame, IP6_ERROR_ADJACENCY_PUNT); }
1040
1041 VLIB_REGISTER_NODE (ip6_drop_node,static) = {
1042   .function = ip6_drop,
1043   .name = "ip6-drop",
1044   .vector_size = sizeof (u32),
1045
1046   .format_trace = format_ip6_forward_next_trace,
1047
1048   .n_next_nodes = 1,
1049   .next_nodes = {
1050     [0] = "error-drop",
1051   },
1052 };
1053
1054 VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop)
1055
1056 VLIB_REGISTER_NODE (ip6_punt_node,static) = {
1057   .function = ip6_punt,
1058   .name = "ip6-punt",
1059   .vector_size = sizeof (u32),
1060
1061   .format_trace = format_ip6_forward_next_trace,
1062
1063   .n_next_nodes = 1,
1064   .next_nodes = {
1065     [0] = "error-punt",
1066   },
1067 };
1068
1069 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt)
1070
1071 VLIB_REGISTER_NODE (ip6_multicast_node,static) = {
1072   .function = ip6_drop,
1073   .name = "ip6-multicast",
1074   .vector_size = sizeof (u32),
1075
1076   .format_trace = format_ip6_forward_next_trace,
1077
1078   .n_next_nodes = 1,
1079   .next_nodes = {
1080     [0] = "error-drop",
1081   },
1082 };
1083
1084 /* Compute TCP/UDP/ICMP6 checksum in software. */
1085 u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6_header_t * ip0, int *bogus_lengthp)
1086 {
1087   ip_csum_t sum0;
1088   u16 sum16, payload_length_host_byte_order;
1089   u32 i, n_this_buffer, n_bytes_left;
1090   u32 headers_size = sizeof(ip0[0]);
1091   void * data_this_buffer;
1092
1093   ASSERT(bogus_lengthp);
1094   *bogus_lengthp = 0;
1095
1096   /* Initialize checksum with ip header. */
1097   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
1098   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
1099   data_this_buffer = (void *) (ip0 + 1);
1100
1101   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
1102     {
1103       sum0 = ip_csum_with_carry (sum0,
1104                                  clib_mem_unaligned (&ip0->src_address.as_uword[i], uword));
1105       sum0 = ip_csum_with_carry (sum0,
1106                                  clib_mem_unaligned (&ip0->dst_address.as_uword[i], uword));
1107     }
1108
1109   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1110   if (PREDICT_FALSE (ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
1111     {
1112       u32  skip_bytes;
1113       ip6_hop_by_hop_ext_t *ext_hdr = (ip6_hop_by_hop_ext_t  *)data_this_buffer;
1114
1115       /* validate really icmp6 next */
1116       ASSERT(ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
1117
1118       skip_bytes = 8* (1 + ext_hdr->n_data_u64s);
1119       data_this_buffer  = (void *)((u8 *)data_this_buffer + skip_bytes);
1120
1121       payload_length_host_byte_order  -= skip_bytes;
1122       headers_size += skip_bytes;
1123    }
1124
1125   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1126 #if DPDK > 0
1127   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1128   {
1129     struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
1130     u8 nb_segs = mb->nb_segs;
1131
1132     n_this_buffer = (p0->current_length > headers_size ?
1133                      p0->current_length - headers_size : 0);
1134     while (n_bytes_left)
1135       {
1136         sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1137         n_bytes_left -= n_this_buffer;
1138
1139         mb = mb->next;
1140         nb_segs--;
1141         if ((nb_segs == 0) || (mb == 0))
1142           break;
1143
1144         data_this_buffer = rte_ctrlmbuf_data(mb);
1145         n_this_buffer = mb->data_len;
1146       }
1147     if (n_bytes_left || nb_segs)
1148       {
1149         *bogus_lengthp = 1;
1150         return 0xfefe;
1151       }
1152   }
1153   else sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1154 #else
1155   if (p0 && n_this_buffer + headers_size  > p0->current_length)
1156     n_this_buffer = p0->current_length > headers_size  ? p0->current_length - headers_size  : 0;
1157   while (1)
1158     {
1159       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1160       n_bytes_left -= n_this_buffer;
1161       if (n_bytes_left == 0)
1162         break;
1163
1164       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
1165         {
1166           *bogus_lengthp = 1;
1167           return 0xfefe;
1168         }
1169       p0 = vlib_get_buffer (vm, p0->next_buffer);
1170       data_this_buffer = vlib_buffer_get_current (p0);
1171       n_this_buffer = p0->current_length;
1172     }
1173 #endif /* DPDK */
1174
1175   sum16 = ~ ip_csum_fold (sum0);
1176
1177   return sum16;
1178 }
1179
1180 u32 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
1181 {
1182   ip6_header_t * ip0 = vlib_buffer_get_current (p0);
1183   udp_header_t * udp0;
1184   u16 sum16;
1185   int bogus_length;
1186
1187   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
1188   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1189           || ip0->protocol == IP_PROTOCOL_ICMP6
1190           || ip0->protocol == IP_PROTOCOL_UDP
1191           || ip0->protocol ==  IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
1192
1193   udp0 = (void *) (ip0 + 1);
1194   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1195     {
1196       p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1197                     | IP_BUFFER_L4_CHECKSUM_CORRECT);
1198       return p0->flags;
1199     }
1200
1201   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
1202
1203   p0->flags |= (IP_BUFFER_L4_CHECKSUM_COMPUTED
1204                 | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1205
1206   return p0->flags;
1207 }
1208
1209 static uword
1210 ip6_local (vlib_main_t * vm,
1211            vlib_node_runtime_t * node,
1212            vlib_frame_t * frame)
1213 {
1214   ip6_main_t * im = &ip6_main;
1215   ip_lookup_main_t * lm = &im->lookup_main;
1216   ip_local_next_t next_index;
1217   u32 * from, * to_next, n_left_from, n_left_to_next;
1218   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1219
1220   from = vlib_frame_vector_args (frame);
1221   n_left_from = frame->n_vectors;
1222   next_index = node->cached_next_index;
1223
1224   if (node->flags & VLIB_NODE_FLAG_TRACE)
1225     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1226
1227   while (n_left_from > 0)
1228     {
1229       vlib_get_next_frame (vm, node, next_index,
1230                            to_next, n_left_to_next);
1231
1232       while (n_left_from >= 4 && n_left_to_next >= 2)
1233         {
1234           vlib_buffer_t * p0, * p1;
1235           ip6_header_t * ip0, * ip1;
1236           udp_header_t * udp0, * udp1;
1237           u32 pi0, ip_len0, udp_len0, flags0, next0;
1238           u32 pi1, ip_len1, udp_len1, flags1, next1;
1239           i32 len_diff0, len_diff1;
1240           u8 error0, type0, good_l4_checksum0;
1241           u8 error1, type1, good_l4_checksum1;
1242
1243           pi0 = to_next[0] = from[0];
1244           pi1 = to_next[1] = from[1];
1245           from += 2;
1246           n_left_from -= 2;
1247           to_next += 2;
1248           n_left_to_next -= 2;
1249
1250           p0 = vlib_get_buffer (vm, pi0);
1251           p1 = vlib_get_buffer (vm, pi1);
1252
1253           ip0 = vlib_buffer_get_current (p0);
1254           ip1 = vlib_buffer_get_current (p1);
1255
1256           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1257           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1258
1259           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1260           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1261
1262           flags0 = p0->flags;
1263           flags1 = p1->flags;
1264
1265           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1266           good_l4_checksum1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1267
1268           udp0 = ip6_next_header (ip0);
1269           udp1 = ip6_next_header (ip1);
1270
1271           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1272           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1273           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP && udp1->checksum == 0;
1274
1275           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1276           good_l4_checksum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1277
1278           /* Verify UDP length. */
1279           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1280           ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1281           udp_len0 = clib_net_to_host_u16 (udp0->length);
1282           udp_len1 = clib_net_to_host_u16 (udp1->length);
1283
1284           len_diff0 = ip_len0 - udp_len0;
1285           len_diff1 = ip_len1 - udp_len1;
1286
1287           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1288           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1289
1290           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1291                              && ! good_l4_checksum0
1292                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1293             {
1294               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1295               good_l4_checksum0 =
1296                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1297             }
1298           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1299                              && ! good_l4_checksum1
1300                              && ! (flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1301             {
1302               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1303               good_l4_checksum1 =
1304                 (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1305             }
1306
1307           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1308
1309           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1310           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1311
1312           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1313           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1314           error0 = (! good_l4_checksum0
1315                     ? IP6_ERROR_UDP_CHECKSUM + type0
1316                     : error0);
1317           error1 = (! good_l4_checksum1
1318                     ? IP6_ERROR_UDP_CHECKSUM + type1
1319                     : error1);
1320
1321           /* Drop packets from unroutable hosts. */
1322           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1323           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1324               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1325               !ip6_address_is_link_local_unicast(&ip0->src_address))
1326             {
1327               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1328               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1329                         ? IP6_ERROR_SRC_LOOKUP_MISS
1330                         : error0);
1331             }
1332           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1333               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1334               !ip6_address_is_link_local_unicast(&ip1->src_address))
1335             {
1336               u32 src_adj_index1 = ip6_src_lookup_for_packet (im, p1, ip1);
1337               error1 = (ADJ_INDEX_INVALID == src_adj_index1
1338                         ? IP6_ERROR_SRC_LOOKUP_MISS
1339                         : error1);
1340             }
1341
1342           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1343           next1 = error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1344
1345           p0->error = error_node->errors[error0];
1346           p1->error = error_node->errors[error1];
1347
1348           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1349                                            to_next, n_left_to_next,
1350                                            pi0, pi1, next0, next1);
1351         }
1352
1353       while (n_left_from > 0 && n_left_to_next > 0)
1354         {
1355           vlib_buffer_t * p0;
1356           ip6_header_t * ip0;
1357           udp_header_t * udp0;
1358           u32 pi0, ip_len0, udp_len0, flags0, next0;
1359           i32 len_diff0;
1360           u8 error0, type0, good_l4_checksum0;
1361
1362           pi0 = to_next[0] = from[0];
1363           from += 1;
1364           n_left_from -= 1;
1365           to_next += 1;
1366           n_left_to_next -= 1;
1367
1368           p0 = vlib_get_buffer (vm, pi0);
1369
1370           ip0 = vlib_buffer_get_current (p0);
1371
1372           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1373           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1374
1375           flags0 = p0->flags;
1376
1377           good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1378
1379           udp0 = ip6_next_header (ip0);
1380
1381           /* Don't verify UDP checksum for packets with explicit zero checksum. */
1382           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP && udp0->checksum == 0;
1383
1384           good_l4_checksum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1385
1386           /* Verify UDP length. */
1387           ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1388           udp_len0 = clib_net_to_host_u16 (udp0->length);
1389
1390           len_diff0 = ip_len0 - udp_len0;
1391
1392           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1393
1394           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1395                              && ! good_l4_checksum0
1396                              && ! (flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED)))
1397             {
1398               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1399               good_l4_checksum0 =
1400                 (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1401             }
1402
1403           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1404
1405           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1406
1407           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM);
1408           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP == IP6_ERROR_ICMP_CHECKSUM);
1409           error0 = (! good_l4_checksum0
1410                     ? IP6_ERROR_UDP_CHECKSUM + type0
1411                     : error0);
1412
1413           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1414           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1415               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1416               !ip6_address_is_link_local_unicast(&ip0->src_address))
1417             {
1418               u32 src_adj_index0 = ip6_src_lookup_for_packet (im, p0, ip0);
1419               error0 = (ADJ_INDEX_INVALID == src_adj_index0
1420                         ? IP6_ERROR_SRC_LOOKUP_MISS
1421                         : error0);
1422             }
1423
1424           next0 = error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1425
1426           p0->error = error_node->errors[error0];
1427
1428           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1429                                            to_next, n_left_to_next,
1430                                            pi0, next0);
1431         }
1432
1433       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1434     }
1435
1436   return frame->n_vectors;
1437 }
1438
1439 VLIB_REGISTER_NODE (ip6_local_node,static) = {
1440   .function = ip6_local,
1441   .name = "ip6-local",
1442   .vector_size = sizeof (u32),
1443
1444   .format_trace = format_ip6_forward_next_trace,
1445
1446   .n_next_nodes = IP_LOCAL_N_NEXT,
1447   .next_nodes = {
1448     [IP_LOCAL_NEXT_DROP] = "error-drop",
1449     [IP_LOCAL_NEXT_PUNT] = "error-punt",
1450     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1451     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1452   },
1453 };
1454
1455 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local)
1456
1457 void ip6_register_protocol (u32 protocol, u32 node_index)
1458 {
1459   vlib_main_t * vm = vlib_get_main();
1460   ip6_main_t * im = &ip6_main;
1461   ip_lookup_main_t * lm = &im->lookup_main;
1462
1463   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1464   lm->local_next_by_ip_protocol[protocol] = vlib_node_add_next (vm, ip6_local_node.index, node_index);
1465 }
1466
1467 typedef enum {
1468   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1469   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1470   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1471 } ip6_discover_neighbor_next_t;
1472
1473 typedef enum {
1474   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1475   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1476   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
1477 } ip6_discover_neighbor_error_t;
1478
1479 static uword
1480 ip6_discover_neighbor_inline (vlib_main_t * vm,
1481                               vlib_node_runtime_t * node,
1482                               vlib_frame_t * frame,
1483                               int is_glean)
1484 {
1485   vnet_main_t * vnm = vnet_get_main();
1486   ip6_main_t * im = &ip6_main;
1487   ip_lookup_main_t * lm = &im->lookup_main;
1488   u32 * from, * to_next_drop;
1489   uword n_left_from, n_left_to_next_drop;
1490   static f64 time_last_seed_change = -1e100;
1491   static u32 hash_seeds[3];
1492   static uword hash_bitmap[256 / BITS (uword)];
1493   f64 time_now;
1494   int bogus_length;
1495
1496   if (node->flags & VLIB_NODE_FLAG_TRACE)
1497     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1498
1499   time_now = vlib_time_now (vm);
1500   if (time_now - time_last_seed_change > 1e-3)
1501     {
1502       uword i;
1503       u32 * r = clib_random_buffer_get_data (&vm->random_buffer,
1504                                              sizeof (hash_seeds));
1505       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1506         hash_seeds[i] = r[i];
1507
1508       /* Mark all hash keys as been not-seen before. */
1509       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1510         hash_bitmap[i] = 0;
1511
1512       time_last_seed_change = time_now;
1513     }
1514
1515   from = vlib_frame_vector_args (frame);
1516   n_left_from = frame->n_vectors;
1517
1518   while (n_left_from > 0)
1519     {
1520       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1521                            to_next_drop, n_left_to_next_drop);
1522
1523       while (n_left_from > 0 && n_left_to_next_drop > 0)
1524         {
1525           vlib_buffer_t * p0;
1526           ip6_header_t * ip0;
1527           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1528           uword bm0;
1529           ip_adjacency_t * adj0;
1530           vnet_hw_interface_t * hw_if0;
1531           u32 next0;
1532
1533           pi0 = from[0];
1534
1535           p0 = vlib_get_buffer (vm, pi0);
1536
1537           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1538
1539           ip0 = vlib_buffer_get_current (p0);
1540
1541           adj0 = ip_get_adjacency (lm, adj_index0);
1542
1543           if (!is_glean)
1544             {
1545               ip0->dst_address.as_u64[0] = adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
1546               ip0->dst_address.as_u64[1] = adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
1547             }
1548
1549           a0 = hash_seeds[0];
1550           b0 = hash_seeds[1];
1551           c0 = hash_seeds[2];
1552
1553           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1554           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1555
1556           a0 ^= sw_if_index0;
1557           b0 ^= ip0->dst_address.as_u32[0];
1558           c0 ^= ip0->dst_address.as_u32[1];
1559
1560           hash_v3_mix32 (a0, b0, c0);
1561
1562           b0 ^= ip0->dst_address.as_u32[2];
1563           c0 ^= ip0->dst_address.as_u32[3];
1564
1565           hash_v3_finalize32 (a0, b0, c0);
1566
1567           c0 &= BITS (hash_bitmap) - 1;
1568           c0 = c0 / BITS (uword);
1569           m0 = (uword) 1 << (c0 % BITS (uword));
1570
1571           bm0 = hash_bitmap[c0];
1572           drop0 = (bm0 & m0) != 0;
1573
1574           /* Mark it as seen. */
1575           hash_bitmap[c0] = bm0 | m0;
1576
1577           from += 1;
1578           n_left_from -= 1;
1579           to_next_drop[0] = pi0;
1580           to_next_drop += 1;
1581           n_left_to_next_drop -= 1;
1582
1583           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1584
1585           /* If the interface is link-down, drop the pkt */
1586           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1587             drop0 = 1;
1588
1589           p0->error =
1590             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1591                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1592           if (drop0)
1593             continue;
1594
1595           {
1596             u32 bi0 = 0;
1597             icmp6_neighbor_solicitation_header_t * h0;
1598             vlib_buffer_t * b0;
1599
1600             h0 = vlib_packet_template_get_packet
1601               (vm, &im->discover_neighbor_packet_template, &bi0);
1602
1603             /*
1604              * Build ethernet header.
1605              * Choose source address based on destination lookup
1606              * adjacency.
1607              */
1608             if (ip6_src_address_for_packet (lm,
1609                                             sw_if_index0,
1610                                             &h0->ip.src_address))
1611               {
1612                 /* There is no address on the interface */
1613                 p0->error = node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
1614                 vlib_buffer_free(vm, &bi0, 1);
1615                 continue;
1616               }
1617
1618             /*
1619              * Destination address is a solicited node multicast address.
1620              * We need to fill in
1621              * the low 24 bits with low 24 bits of target's address.
1622              */
1623             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1624             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1625             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1626
1627             h0->neighbor.target_address = ip0->dst_address;
1628
1629             clib_memcpy (h0->link_layer_option.ethernet_address,
1630                     hw_if0->hw_address, vec_len (hw_if0->hw_address));
1631
1632             /* $$$$ appears we need this; why is the checksum non-zero? */
1633             h0->neighbor.icmp.checksum = 0;
1634             h0->neighbor.icmp.checksum =
1635               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1636                                                  &bogus_length);
1637
1638             ASSERT (bogus_length == 0);
1639
1640             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1641             b0 = vlib_get_buffer (vm, bi0);
1642             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1643               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1644
1645             /* Add rewrite/encap string. */
1646             vnet_rewrite_one_header (adj0[0], h0,
1647                                      sizeof (ethernet_header_t));
1648             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1649
1650             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1651
1652             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1653           }
1654         }
1655
1656       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1657                            n_left_to_next_drop);
1658     }
1659
1660   return frame->n_vectors;
1661 }
1662
1663 static uword
1664 ip6_discover_neighbor (vlib_main_t * vm,
1665                        vlib_node_runtime_t * node,
1666                        vlib_frame_t * frame)
1667 {
1668     return (ip6_discover_neighbor_inline(vm, node, frame, 0));
1669 }
1670
1671 static uword
1672 ip6_glean (vlib_main_t * vm,
1673            vlib_node_runtime_t * node,
1674            vlib_frame_t * frame)
1675 {
1676     return (ip6_discover_neighbor_inline(vm, node, frame, 1));
1677 }
1678
1679 static char * ip6_discover_neighbor_error_strings[] = {
1680   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1681   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT]
1682   = "neighbor solicitations sent",
1683   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
1684     = "no source address for ND solicitation",
1685 };
1686
1687 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) = {
1688   .function = ip6_discover_neighbor,
1689   .name = "ip6-discover-neighbor",
1690   .vector_size = sizeof (u32),
1691
1692   .format_trace = format_ip6_forward_next_trace,
1693
1694   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1695   .error_strings = ip6_discover_neighbor_error_strings,
1696
1697   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1698   .next_nodes = {
1699     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1700     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1701   },
1702 };
1703
1704 VLIB_REGISTER_NODE (ip6_glean_node) = {
1705   .function = ip6_glean,
1706   .name = "ip6-glean",
1707   .vector_size = sizeof (u32),
1708
1709   .format_trace = format_ip6_forward_next_trace,
1710
1711   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1712   .error_strings = ip6_discover_neighbor_error_strings,
1713
1714   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1715   .next_nodes = {
1716     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "error-drop",
1717     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1718   },
1719 };
1720
1721 clib_error_t *
1722 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
1723 {
1724   vnet_main_t * vnm = vnet_get_main();
1725   ip6_main_t * im = &ip6_main;
1726   icmp6_neighbor_solicitation_header_t * h;
1727   ip6_address_t * src;
1728   ip_interface_address_t * ia;
1729   ip_adjacency_t * adj;
1730   vnet_hw_interface_t * hi;
1731   vnet_sw_interface_t * si;
1732   vlib_buffer_t * b;
1733   u32 bi = 0;
1734   int bogus_length;
1735
1736   si = vnet_get_sw_interface (vnm, sw_if_index);
1737
1738   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1739     {
1740       return clib_error_return (0, "%U: interface %U down",
1741                                 format_ip6_address, dst,
1742                                 format_vnet_sw_if_index_name, vnm,
1743                                 sw_if_index);
1744     }
1745
1746   src = ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1747   if (! src)
1748     {
1749       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1750       return clib_error_return
1751         (0, "no matching interface address for destination %U (interface %U)",
1752          format_ip6_address, dst,
1753          format_vnet_sw_if_index_name, vnm, sw_if_index);
1754     }
1755
1756   h = vlib_packet_template_get_packet (vm, &im->discover_neighbor_packet_template, &bi);
1757
1758   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1759
1760   /* Destination address is a solicited node multicast address.  We need to fill in
1761      the low 24 bits with low 24 bits of target's address. */
1762   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1763   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1764   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1765
1766   h->ip.src_address = src[0];
1767   h->neighbor.target_address = dst[0];
1768
1769   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, vec_len (hi->hw_address));
1770
1771   h->neighbor.icmp.checksum =
1772     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1773   ASSERT(bogus_length == 0);
1774
1775   b = vlib_get_buffer (vm, bi);
1776   vnet_buffer (b)->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1777
1778   /* Add encapsulation string for software interface (e.g. ethernet header). */
1779   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
1780   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1781   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1782
1783   {
1784     vlib_frame_t * f = vlib_get_frame_to_node (vm, hi->output_node_index);
1785     u32 * to_next = vlib_frame_vector_args (f);
1786     to_next[0] = bi;
1787     f->n_vectors = 1;
1788     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1789   }
1790
1791   return /* no error */ 0;
1792 }
1793
1794 typedef enum {
1795   IP6_REWRITE_NEXT_DROP,
1796   IP6_REWRITE_NEXT_ICMP_ERROR,
1797 } ip6_rewrite_next_t;
1798
1799 always_inline uword
1800 ip6_rewrite_inline (vlib_main_t * vm,
1801                     vlib_node_runtime_t * node,
1802                     vlib_frame_t * frame,
1803                     int rewrite_for_locally_received_packets,
1804                     int is_midchain)
1805 {
1806   ip_lookup_main_t * lm = &ip6_main.lookup_main;
1807   u32 * from = vlib_frame_vector_args (frame);
1808   u32 n_left_from, n_left_to_next, * to_next, next_index;
1809   vlib_node_runtime_t * error_node = vlib_node_get_runtime (vm, ip6_input_node.index);
1810   vlib_rx_or_tx_t adj_rx_tx = rewrite_for_locally_received_packets ? VLIB_RX : VLIB_TX;
1811   ip_config_main_t * cm = &lm->feature_config_mains[VNET_IP_TX_FEAT];
1812
1813   n_left_from = frame->n_vectors;
1814   next_index = node->cached_next_index;
1815   u32 cpu_index = os_get_cpu_number();
1816
1817   while (n_left_from > 0)
1818     {
1819       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1820
1821       while (n_left_from >= 4 && n_left_to_next >= 2)
1822         {
1823           ip_adjacency_t * adj0, * adj1;
1824           vlib_buffer_t * p0, * p1;
1825           ip6_header_t * ip0, * ip1;
1826           u32 pi0, rw_len0, next0, error0, adj_index0;
1827           u32 pi1, rw_len1, next1, error1, adj_index1;
1828           u32 tx_sw_if_index0, tx_sw_if_index1;
1829
1830           /* Prefetch next iteration. */
1831           {
1832             vlib_buffer_t * p2, * p3;
1833
1834             p2 = vlib_get_buffer (vm, from[2]);
1835             p3 = vlib_get_buffer (vm, from[3]);
1836
1837             vlib_prefetch_buffer_header (p2, LOAD);
1838             vlib_prefetch_buffer_header (p3, LOAD);
1839
1840             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1841             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1842
1843             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1844             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1845           }
1846
1847           pi0 = to_next[0] = from[0];
1848           pi1 = to_next[1] = from[1];
1849
1850           from += 2;
1851           n_left_from -= 2;
1852           to_next += 2;
1853           n_left_to_next -= 2;
1854
1855           p0 = vlib_get_buffer (vm, pi0);
1856           p1 = vlib_get_buffer (vm, pi1);
1857
1858           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
1859           adj_index1 = vnet_buffer (p1)->ip.adj_index[adj_rx_tx];
1860
1861           /* We should never rewrite a pkt using the MISS adjacency */
1862           ASSERT(adj_index0 && adj_index1);
1863
1864           ip0 = vlib_buffer_get_current (p0);
1865           ip1 = vlib_buffer_get_current (p1);
1866
1867           error0 = error1 = IP6_ERROR_NONE;
1868           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1869
1870           if (! rewrite_for_locally_received_packets)
1871             {
1872               i32 hop_limit0 = ip0->hop_limit, hop_limit1 = ip1->hop_limit;
1873
1874               /* Input node should have reject packets with hop limit 0. */
1875               ASSERT (ip0->hop_limit > 0);
1876               ASSERT (ip1->hop_limit > 0);
1877
1878               hop_limit0 -= 1;
1879               hop_limit1 -= 1;
1880
1881               ip0->hop_limit = hop_limit0;
1882               ip1->hop_limit = hop_limit1;
1883
1884               /*
1885                * If the hop count drops below 1 when forwarding, generate
1886                * an ICMP response.
1887                */
1888               if (PREDICT_FALSE(hop_limit0 <= 0))
1889                 {
1890                   error0 = IP6_ERROR_TIME_EXPIRED;
1891                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1892                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
1893                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
1894                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1895                 }
1896               if (PREDICT_FALSE(hop_limit1 <= 0))
1897                 {
1898                   error1 = IP6_ERROR_TIME_EXPIRED;
1899                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1900                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32)~0;
1901                   icmp6_error_set_vnet_buffer(p1, ICMP6_time_exceeded,
1902                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
1903                 }
1904             }
1905
1906           adj0 = ip_get_adjacency (lm, adj_index0);
1907           adj1 = ip_get_adjacency (lm, adj_index1);
1908
1909           rw_len0 = adj0[0].rewrite_header.data_bytes;
1910           rw_len1 = adj1[0].rewrite_header.data_bytes;
1911           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
1912           vnet_buffer(p1)->ip.save_rewrite_length = rw_len1;
1913
1914           vlib_increment_combined_counter (&adjacency_counters,
1915                                            cpu_index,
1916                                            adj_index0,
1917                                            /* packet increment */ 0,
1918                                            /* byte increment */ rw_len0);
1919           vlib_increment_combined_counter (&adjacency_counters,
1920                                            cpu_index,
1921                                            adj_index1,
1922                                            /* packet increment */ 0,
1923                                            /* byte increment */ rw_len1);
1924
1925           /* Check MTU of outgoing interface. */
1926           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
1927                     ? IP6_ERROR_MTU_EXCEEDED
1928                     : error0);
1929           error1 = (vlib_buffer_length_in_chain (vm, p1) > adj1[0].rewrite_header.max_l3_packet_bytes
1930                     ? IP6_ERROR_MTU_EXCEEDED
1931                     : error1);
1932
1933           /* Don't adjust the buffer for hop count issue; icmp-error node
1934            * wants to see the IP headerr */
1935           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
1936             {
1937               p0->current_data -= rw_len0;
1938               p0->current_length += rw_len0;
1939
1940               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1941               vnet_buffer (p0)->sw_if_index[VLIB_TX] =
1942                   tx_sw_if_index0;
1943               next0 = adj0[0].rewrite_header.next_index;
1944
1945               if (PREDICT_FALSE
1946                   (clib_bitmap_get (lm->tx_sw_if_has_ip_output_features,
1947                                     tx_sw_if_index0)))
1948                 {
1949                   p0->current_config_index =
1950                     vec_elt (cm->config_index_by_sw_if_index,
1951                              tx_sw_if_index0);
1952                   vnet_get_config_data (&cm->config_main,
1953                                         &p0->current_config_index,
1954                                         &next0,
1955                                         /* # bytes of config data */ 0);
1956                 }
1957             }
1958           if (PREDICT_TRUE(error1 == IP6_ERROR_NONE))
1959             {
1960               p1->current_data -= rw_len1;
1961               p1->current_length += rw_len1;
1962
1963               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1964               vnet_buffer (p1)->sw_if_index[VLIB_TX] =
1965                   tx_sw_if_index1;
1966               next1 = adj1[0].rewrite_header.next_index;
1967
1968               if (PREDICT_FALSE
1969                   (clib_bitmap_get (lm->tx_sw_if_has_ip_output_features,
1970                                     tx_sw_if_index1)))
1971                 {
1972                   p1->current_config_index =
1973                     vec_elt (cm->config_index_by_sw_if_index,
1974                              tx_sw_if_index1);
1975                   vnet_get_config_data (&cm->config_main,
1976                                         &p1->current_config_index,
1977                                         &next1,
1978                                         /* # bytes of config data */ 0);
1979                 }
1980             }
1981
1982           /* Guess we are only writing on simple Ethernet header. */
1983           vnet_rewrite_two_headers (adj0[0], adj1[0],
1984                                     ip0, ip1,
1985                                     sizeof (ethernet_header_t));
1986
1987           if (is_midchain)
1988           {
1989               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
1990               adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
1991           }
1992
1993           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1994                                            to_next, n_left_to_next,
1995                                            pi0, pi1, next0, next1);
1996         }
1997
1998       while (n_left_from > 0 && n_left_to_next > 0)
1999         {
2000           ip_adjacency_t * adj0;
2001           vlib_buffer_t * p0;
2002           ip6_header_t * ip0;
2003           u32 pi0, rw_len0;
2004           u32 adj_index0, next0, error0;
2005           u32 tx_sw_if_index0;
2006
2007           pi0 = to_next[0] = from[0];
2008
2009           p0 = vlib_get_buffer (vm, pi0);
2010
2011           adj_index0 = vnet_buffer (p0)->ip.adj_index[adj_rx_tx];
2012
2013           /* We should never rewrite a pkt using the MISS adjacency */
2014           ASSERT(adj_index0);
2015
2016           adj0 = ip_get_adjacency (lm, adj_index0);
2017
2018           ip0 = vlib_buffer_get_current (p0);
2019
2020           error0 = IP6_ERROR_NONE;
2021           next0 = IP6_REWRITE_NEXT_DROP;
2022
2023           /* Check hop limit */
2024           if (! rewrite_for_locally_received_packets)
2025             {
2026               i32 hop_limit0 = ip0->hop_limit;
2027
2028               ASSERT (ip0->hop_limit > 0);
2029
2030               hop_limit0 -= 1;
2031
2032               ip0->hop_limit = hop_limit0;
2033
2034               if (PREDICT_FALSE(hop_limit0 <= 0))
2035                 {
2036                   /*
2037                    * If the hop count drops below 1 when forwarding, generate
2038                    * an ICMP response.
2039                    */
2040                   error0 = IP6_ERROR_TIME_EXPIRED;
2041                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2042                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32)~0;
2043                   icmp6_error_set_vnet_buffer(p0, ICMP6_time_exceeded,
2044                         ICMP6_time_exceeded_ttl_exceeded_in_transit, 0);
2045                 }
2046             }
2047
2048           /* Guess we are only writing on simple Ethernet header. */
2049           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2050
2051           /* Update packet buffer attributes/set output interface. */
2052           rw_len0 = adj0[0].rewrite_header.data_bytes;
2053           vnet_buffer(p0)->ip.save_rewrite_length = rw_len0;
2054
2055           vlib_increment_combined_counter (&adjacency_counters,
2056                                            cpu_index,
2057                                            adj_index0,
2058                                            /* packet increment */ 0,
2059                                            /* byte increment */ rw_len0);
2060
2061           /* Check MTU of outgoing interface. */
2062           error0 = (vlib_buffer_length_in_chain (vm, p0) > adj0[0].rewrite_header.max_l3_packet_bytes
2063                     ? IP6_ERROR_MTU_EXCEEDED
2064                     : error0);
2065
2066           /* Don't adjust the buffer for hop count issue; icmp-error node
2067            * wants to see the IP headerr */
2068           if (PREDICT_TRUE(error0 == IP6_ERROR_NONE))
2069             {
2070               p0->current_data -= rw_len0;
2071               p0->current_length += rw_len0;
2072
2073               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2074
2075               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2076               next0 = adj0[0].rewrite_header.next_index;
2077
2078               if (PREDICT_FALSE
2079                   (clib_bitmap_get (lm->tx_sw_if_has_ip_output_features,
2080                                     tx_sw_if_index0)))
2081                   {
2082                     p0->current_config_index =
2083                       vec_elt (cm->config_index_by_sw_if_index,
2084                                tx_sw_if_index0);
2085                     vnet_get_config_data (&cm->config_main,
2086                                           &p0->current_config_index,
2087                                           &next0,
2088                                           /* # bytes of config data */ 0);
2089                   }
2090             }
2091
2092           if (is_midchain)
2093           {
2094               adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
2095           }
2096
2097           p0->error = error_node->errors[error0];
2098
2099           from += 1;
2100           n_left_from -= 1;
2101           to_next += 1;
2102           n_left_to_next -= 1;
2103
2104           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2105                                            to_next, n_left_to_next,
2106                                            pi0, next0);
2107         }
2108
2109       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2110     }
2111
2112   /* Need to do trace after rewrites to pick up new packet data. */
2113   if (node->flags & VLIB_NODE_FLAG_TRACE)
2114     ip6_forward_next_trace (vm, node, frame, adj_rx_tx);
2115
2116   return frame->n_vectors;
2117 }
2118
2119 static uword
2120 ip6_rewrite_transit (vlib_main_t * vm,
2121                      vlib_node_runtime_t * node,
2122                      vlib_frame_t * frame)
2123 {
2124   return ip6_rewrite_inline (vm, node, frame,
2125                              /* rewrite_for_locally_received_packets */ 0,
2126                              /* midchain */ 0);
2127 }
2128
2129 static uword
2130 ip6_rewrite_local (vlib_main_t * vm,
2131                    vlib_node_runtime_t * node,
2132                    vlib_frame_t * frame)
2133 {
2134   return ip6_rewrite_inline (vm, node, frame,
2135                              /* rewrite_for_locally_received_packets */ 1,
2136                              /* midchain */ 0);
2137 }
2138
2139 static uword
2140 ip6_midchain (vlib_main_t * vm,
2141               vlib_node_runtime_t * node,
2142               vlib_frame_t * frame)
2143 {
2144   return ip6_rewrite_inline (vm, node, frame,
2145                              /* rewrite_for_locally_received_packets */ 0,
2146                              /* midchain */ 1);
2147 }
2148
2149 VLIB_REGISTER_NODE (ip6_midchain_node) = {
2150   .function = ip6_midchain,
2151   .name = "ip6-midchain",
2152   .vector_size = sizeof (u32),
2153
2154   .format_trace = format_ip6_forward_next_trace,
2155
2156   .sibling_of = "ip6-rewrite",
2157
2158   .next_nodes = {
2159     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2160   },
2161 };
2162
2163 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain)
2164
2165 VLIB_REGISTER_NODE (ip6_rewrite_node) = {
2166   .function = ip6_rewrite_transit,
2167   .name = "ip6-rewrite",
2168   .vector_size = sizeof (u32),
2169
2170   .format_trace = format_ip6_rewrite_trace,
2171
2172   .n_next_nodes = 2,
2173   .next_nodes = {
2174     [IP6_REWRITE_NEXT_DROP] = "error-drop",
2175     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2176   },
2177 };
2178
2179 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit);
2180
2181 VLIB_REGISTER_NODE (ip6_rewrite_local_node) = {
2182   .function = ip6_rewrite_local,
2183   .name = "ip6-rewrite-local",
2184   .vector_size = sizeof (u32),
2185
2186   .sibling_of = "ip6-rewrite",
2187
2188   .format_trace = format_ip6_rewrite_trace,
2189
2190   .n_next_nodes = 0,
2191 };
2192
2193 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local);
2194
2195 /*
2196  * Hop-by-Hop handling
2197  */
2198
2199 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2200
2201 #define foreach_ip6_hop_by_hop_error \
2202 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2203 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2204 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2205
2206 typedef enum {
2207 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2208   foreach_ip6_hop_by_hop_error
2209 #undef _
2210   IP6_HOP_BY_HOP_N_ERROR,
2211 } ip6_hop_by_hop_error_t;
2212
2213 /*
2214  * Primary h-b-h handler trace support
2215  * We work pretty hard on the problem for obvious reasons
2216  */
2217 typedef struct {
2218   u32 next_index;
2219   u32 trace_len;
2220   u8 option_data[256];
2221 } ip6_hop_by_hop_trace_t;
2222
2223 vlib_node_registration_t ip6_hop_by_hop_node;
2224
2225 static char * ip6_hop_by_hop_error_strings[] = {
2226 #define _(sym,string) string,
2227   foreach_ip6_hop_by_hop_error
2228 #undef _
2229 };
2230
2231 static u8 *
2232 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2233 {
2234   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2235   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2236   ip6_hop_by_hop_trace_t * t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2237   ip6_hop_by_hop_header_t *hbh0;
2238   ip6_hop_by_hop_option_t *opt0, *limit0;
2239   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2240
2241   u8 type0;
2242
2243   hbh0 = (ip6_hop_by_hop_header_t *)t->option_data;
2244
2245   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2246               t->next_index, (hbh0->length+1)<<3, t->trace_len);
2247
2248   opt0 = (ip6_hop_by_hop_option_t *) (hbh0+1);
2249   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *)hbh0) + t->trace_len;
2250
2251   while (opt0 < limit0) {
2252     type0 = opt0->type;
2253     switch (type0) {
2254     case 0: /* Pad, just stop */
2255       opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2256       break;
2257
2258     default:
2259       if (hm->trace[type0]) {
2260         s = (*hm->trace[type0])(s, opt0);
2261       } else {
2262         s = format (s, "\n    unrecognized option %d length %d", type0, opt0->length);
2263       }
2264       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2265       break;
2266     }
2267   }
2268   return s;
2269 }
2270
2271 always_inline u8 ip6_scan_hbh_options (
2272                                        vlib_buffer_t * b0,
2273                                        ip6_header_t *ip0,
2274                                        ip6_hop_by_hop_header_t *hbh0,
2275                                        ip6_hop_by_hop_option_t *opt0,
2276                                        ip6_hop_by_hop_option_t *limit0,
2277                                        u32 *next0)
2278 {
2279   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2280   u8 type0;
2281   u8 error0 = 0;
2282
2283   while (opt0 < limit0)
2284     {
2285       type0 = opt0->type;
2286       switch (type0)
2287         {
2288         case 0: /* Pad1 */
2289           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *)opt0) + 1;
2290           continue;
2291         case 1: /* PadN */
2292           break;
2293         default:
2294           if (hm->options[type0])
2295             {
2296               if ((*hm->options[type0])(b0, ip0, opt0) < 0)
2297                 {
2298                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2299                   return(error0);
2300                 }
2301             }
2302           else
2303             {
2304               /* Unrecognized mandatory option, check the two high order bits */
2305               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2306                 {
2307                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2308                   break;
2309                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2310                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2311                   *next0 = IP_LOOKUP_NEXT_DROP;
2312                   break;
2313                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2314                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2315                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2316                   icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2317                                               ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2318                   break;
2319                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2320                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2321                   if (!ip6_address_is_multicast(&ip0->dst_address))
2322                     {
2323                       *next0 =  IP_LOOKUP_NEXT_ICMP_ERROR;
2324                       icmp6_error_set_vnet_buffer(b0, ICMP6_parameter_problem,
2325                                                   ICMP6_parameter_problem_unrecognized_option, (u8 *)opt0 - (u8 *)ip0);
2326                     }
2327                   else
2328                     {
2329                       *next0 =  IP_LOOKUP_NEXT_DROP;
2330                     }
2331                   break;
2332                 }
2333               return(error0);
2334             }
2335         }
2336       opt0 = (ip6_hop_by_hop_option_t *) (((u8 *)opt0) + opt0->length + sizeof (ip6_hop_by_hop_option_t));
2337     }
2338   return(error0);
2339 }
2340
2341 /*
2342  * Process the Hop-by-Hop Options header
2343  */
2344 static uword
2345 ip6_hop_by_hop (vlib_main_t * vm,
2346                 vlib_node_runtime_t * node,
2347                 vlib_frame_t * frame)
2348 {
2349   vlib_node_runtime_t *error_node = vlib_node_get_runtime(vm, ip6_hop_by_hop_node.index);
2350   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2351   u32 n_left_from, *from, *to_next;
2352   ip_lookup_next_t next_index;
2353   ip6_main_t * im = &ip6_main;
2354   ip_lookup_main_t *lm = &im->lookup_main;
2355
2356   from = vlib_frame_vector_args (frame);
2357   n_left_from = frame->n_vectors;
2358   next_index = node->cached_next_index;
2359
2360   while (n_left_from > 0) {
2361     u32 n_left_to_next;
2362
2363     vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2364
2365     while (n_left_from >= 4 && n_left_to_next >= 2) {
2366       u32 bi0, bi1;
2367       vlib_buffer_t * b0, *b1;
2368       u32 next0, next1;
2369       ip6_header_t * ip0, *ip1;
2370       ip6_hop_by_hop_header_t *hbh0, *hbh1;
2371       ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2372       u8 error0 = 0, error1 = 0;
2373
2374       /* Prefetch next iteration. */
2375       {
2376         vlib_buffer_t * p2, * p3;
2377
2378         p2 = vlib_get_buffer (vm, from[2]);
2379         p3 = vlib_get_buffer (vm, from[3]);
2380
2381         vlib_prefetch_buffer_header (p2, LOAD);
2382         vlib_prefetch_buffer_header (p3, LOAD);
2383
2384         CLIB_PREFETCH (p2->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2385         CLIB_PREFETCH (p3->data, 2*CLIB_CACHE_LINE_BYTES, LOAD);
2386       }
2387
2388       /* Speculatively enqueue b0, b1 to the current next frame */
2389       to_next[0] = bi0 = from[0];
2390       to_next[1] = bi1 = from[1];
2391       from += 2;
2392       to_next += 2;
2393       n_left_from -= 2;
2394       n_left_to_next -= 2;
2395
2396       b0 = vlib_get_buffer (vm, bi0);
2397       b1 = vlib_get_buffer (vm, bi1);
2398       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2399       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2400       u32 adj_index1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
2401       ip_adjacency_t *adj1 = ip_get_adjacency(lm, adj_index1);
2402
2403       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2404       next0 = adj0->lookup_next_index;
2405       next1 = adj1->lookup_next_index;
2406
2407       ip0 = vlib_buffer_get_current (b0);
2408       ip1 = vlib_buffer_get_current (b1);
2409       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2410       hbh1 = (ip6_hop_by_hop_header_t *)(ip1+1);
2411       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2412       opt1 = (ip6_hop_by_hop_option_t *)(hbh1+1);
2413       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2414       limit1 = (ip6_hop_by_hop_option_t *)((u8 *)hbh1 + ((hbh1->length + 1) << 3));
2415
2416       /*
2417        * Basic validity checks
2418        */
2419       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2420         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2421         next0 = IP_LOOKUP_NEXT_DROP;
2422         goto outdual;
2423       }
2424       /* Scan the set of h-b-h options, process ones that we understand */
2425       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2426
2427       if ((hbh1->length + 1) << 3 > clib_net_to_host_u16(ip1->payload_length)) {
2428         error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2429         next1 = IP_LOOKUP_NEXT_DROP;
2430         goto outdual;
2431       }
2432       /* Scan the set of h-b-h options, process ones that we understand */
2433       error1 = ip6_scan_hbh_options(b1,ip1,hbh1,opt1,limit1, &next1);
2434
2435     outdual:
2436       /* Has the classifier flagged this buffer for special treatment? */
2437       if ((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index == OI_DECAP))
2438         next0 = hm->next_override;
2439
2440       /* Has the classifier flagged this buffer for special treatment? */
2441       if ((error1 == 0) && (vnet_buffer(b1)->l2_classify.opaque_index == OI_DECAP))
2442         next1 = hm->next_override;
2443
2444       if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
2445         {
2446           if (b0->flags & VLIB_BUFFER_IS_TRACED) {
2447             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2448             u32 trace_len = (hbh0->length + 1) << 3;
2449             t->next_index = next0;
2450             /* Capture the h-b-h option verbatim */
2451             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2452             t->trace_len = trace_len;
2453             clib_memcpy(t->option_data, hbh0, trace_len);
2454           }
2455           if (b1->flags & VLIB_BUFFER_IS_TRACED) {
2456             ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b1, sizeof (*t));
2457             u32 trace_len = (hbh1->length + 1) << 3;
2458             t->next_index = next1;
2459             /* Capture the h-b-h option verbatim */
2460             trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2461             t->trace_len = trace_len;
2462             clib_memcpy(t->option_data, hbh1, trace_len);
2463           }
2464
2465         }
2466
2467       b0->error = error_node->errors[error0];
2468       b1->error = error_node->errors[error1];
2469
2470       /* verify speculative enqueue, maybe switch current next frame */
2471       vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0,
2472                                        bi1,next0, next1);
2473     }
2474
2475     while (n_left_from > 0 && n_left_to_next > 0) {
2476       u32 bi0;
2477       vlib_buffer_t * b0;
2478       u32 next0;
2479       ip6_header_t * ip0;
2480       ip6_hop_by_hop_header_t *hbh0;
2481       ip6_hop_by_hop_option_t *opt0, *limit0;
2482       u8 error0 = 0;
2483
2484       /* Speculatively enqueue b0 to the current next frame */
2485       bi0 = from[0];
2486       to_next[0] = bi0;
2487       from += 1;
2488       to_next += 1;
2489       n_left_from -= 1;
2490       n_left_to_next -= 1;
2491
2492       b0 = vlib_get_buffer (vm, bi0);
2493       u32 adj_index0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
2494       ip_adjacency_t *adj0 = ip_get_adjacency(lm, adj_index0);
2495       /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2496       next0 = adj0->lookup_next_index;
2497
2498       ip0 = vlib_buffer_get_current (b0);
2499       hbh0 = (ip6_hop_by_hop_header_t *)(ip0+1);
2500       opt0 = (ip6_hop_by_hop_option_t *)(hbh0+1);
2501       limit0 = (ip6_hop_by_hop_option_t *)((u8 *)hbh0 + ((hbh0->length + 1) << 3));
2502
2503       /*
2504        * Basic validity checks
2505        */
2506       if ((hbh0->length + 1) << 3 > clib_net_to_host_u16(ip0->payload_length)) {
2507         error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2508         next0 = IP_LOOKUP_NEXT_DROP;
2509         goto out0;
2510       }
2511
2512       /* Scan the set of h-b-h options, process ones that we understand */
2513       error0 = ip6_scan_hbh_options(b0, ip0, hbh0, opt0, limit0, &next0);
2514
2515     out0:
2516       /* Has the classifier flagged this buffer for special treatment? */
2517       if ((error0 == 0) && (vnet_buffer(b0)->l2_classify.opaque_index == OI_DECAP))
2518         next0 = hm->next_override;
2519
2520       if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) {
2521         ip6_hop_by_hop_trace_t *t = vlib_add_trace(vm, node, b0, sizeof (*t));
2522         u32 trace_len = (hbh0->length + 1) << 3;
2523         t->next_index = next0;
2524         /* Capture the h-b-h option verbatim */
2525         trace_len = trace_len < ARRAY_LEN(t->option_data) ? trace_len : ARRAY_LEN(t->option_data);
2526         t->trace_len = trace_len;
2527         clib_memcpy(t->option_data, hbh0, trace_len);
2528       }
2529
2530       b0->error = error_node->errors[error0];
2531
2532       /* verify speculative enqueue, maybe switch current next frame */
2533       vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0);
2534     }
2535     vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2536   }
2537   return frame->n_vectors;
2538 }
2539
2540 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = {
2541   .function = ip6_hop_by_hop,
2542   .name = "ip6-hop-by-hop",
2543   .sibling_of = "ip6-lookup",
2544   .vector_size = sizeof (u32),
2545   .format_trace = format_ip6_hop_by_hop_trace,
2546   .type = VLIB_NODE_TYPE_INTERNAL,
2547   .n_errors = ARRAY_LEN(ip6_hop_by_hop_error_strings),
2548   .error_strings = ip6_hop_by_hop_error_strings,
2549   .n_next_nodes = 0,
2550 };
2551
2552 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2553
2554 static clib_error_t *
2555 ip6_hop_by_hop_init (vlib_main_t * vm)
2556 {
2557   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2558   memset(hm->options, 0, sizeof(hm->options));
2559   memset(hm->trace, 0, sizeof(hm->trace));
2560   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2561   return (0);
2562 }
2563
2564 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2565
2566 void ip6_hbh_set_next_override (uword next)
2567 {
2568   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2569
2570   hm->next_override = next;
2571 }
2572
2573 int
2574 ip6_hbh_register_option (u8 option,
2575                          int options(vlib_buffer_t *b, ip6_header_t *ip, ip6_hop_by_hop_option_t *opt),
2576                          u8 *trace(u8 *s, ip6_hop_by_hop_option_t *opt))
2577 {
2578   ip6_main_t * im = &ip6_main;
2579   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2580
2581   ASSERT (option < ARRAY_LEN (hm->options));
2582
2583   /* Already registered */
2584   if (hm->options[option])
2585     return (-1);
2586
2587   hm->options[option] = options;
2588   hm->trace[option] = trace;
2589
2590   /* Set global variable */
2591   im->hbh_enabled = 1;
2592
2593   return (0);
2594 }
2595
2596 int
2597 ip6_hbh_unregister_option (u8 option)
2598 {
2599   ip6_main_t * im = &ip6_main;
2600   ip6_hop_by_hop_main_t * hm = &ip6_hop_by_hop_main;
2601
2602   ASSERT (option < ARRAY_LEN (hm->options));
2603
2604   /* Not registered */
2605   if (!hm->options[option])
2606     return (-1);
2607
2608   hm->options[option] = NULL;
2609   hm->trace[option] = NULL;
2610
2611   /* Disable global knob if this was the last option configured */
2612   int i;
2613   bool found = false;
2614   for (i = 0; i < 256; i++) {
2615     if (hm->options[option]) {
2616       found = true;
2617       break;
2618     }
2619   }
2620   if (!found)
2621     im->hbh_enabled = 0;
2622
2623   return (0);
2624 }
2625
2626 /* Global IP6 main. */
2627 ip6_main_t ip6_main;
2628
2629 static clib_error_t *
2630 ip6_lookup_init (vlib_main_t * vm)
2631 {
2632   ip6_main_t * im = &ip6_main;
2633   clib_error_t * error;
2634   uword i;
2635
2636   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2637     {
2638       u32 j, i0, i1;
2639
2640       i0 = i / 32;
2641       i1 = i % 32;
2642
2643       for (j = 0; j < i0; j++)
2644         im->fib_masks[i].as_u32[j] = ~0;
2645
2646       if (i1)
2647         im->fib_masks[i].as_u32[i0] = clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2648     }
2649
2650   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2651
2652   if (im->lookup_table_nbuckets == 0)
2653     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2654
2655   im->lookup_table_nbuckets = 1<< max_log2 (im->lookup_table_nbuckets);
2656
2657   if (im->lookup_table_size == 0)
2658     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2659
2660   BV(clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2661                         "ip6 FIB fwding table",
2662                         im->lookup_table_nbuckets,
2663                         im->lookup_table_size);
2664   BV(clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2665                         "ip6 FIB non-fwding table",
2666                         im->lookup_table_nbuckets,
2667                         im->lookup_table_size);
2668
2669   /* Create FIB with index 0 and table id of 0. */
2670   fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 0);
2671
2672   {
2673     pg_node_t * pn;
2674     pn = pg_get_node (ip6_lookup_node.index);
2675     pn->unformat_edit = unformat_pg_ip6_header;
2676   }
2677
2678   /* Unless explicitly configured, don't process HBH options */
2679   im->hbh_enabled = 0;
2680
2681   {
2682     icmp6_neighbor_solicitation_header_t p;
2683
2684     memset (&p, 0, sizeof (p));
2685
2686     p.ip.ip_version_traffic_class_and_flow_label = clib_host_to_net_u32 (0x6 << 28);
2687     p.ip.payload_length = clib_host_to_net_u16 (sizeof (p)
2688                                                 - STRUCT_OFFSET_OF (icmp6_neighbor_solicitation_header_t, neighbor));
2689     p.ip.protocol = IP_PROTOCOL_ICMP6;
2690     p.ip.hop_limit = 255;
2691     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2692
2693     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2694
2695     p.link_layer_option.header.type = ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2696     p.link_layer_option.header.n_data_u64s = sizeof (p.link_layer_option) / sizeof (u64);
2697
2698     vlib_packet_template_init (vm,
2699                                &im->discover_neighbor_packet_template,
2700                                &p, sizeof (p),
2701                                /* alloc chunk size */ 8,
2702                                "ip6 neighbor discovery");
2703   }
2704
2705   error = ip6_feature_init (vm, im);
2706
2707   return error;
2708 }
2709
2710 VLIB_INIT_FUNCTION (ip6_lookup_init);
2711
2712 static clib_error_t *
2713 add_del_ip6_interface_table (vlib_main_t * vm,
2714                              unformat_input_t * input,
2715                              vlib_cli_command_t * cmd)
2716 {
2717   vnet_main_t * vnm = vnet_get_main();
2718   clib_error_t * error = 0;
2719   u32 sw_if_index, table_id;
2720
2721   sw_if_index = ~0;
2722
2723   if (! unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2724     {
2725       error = clib_error_return (0, "unknown interface `%U'",
2726                                  format_unformat_error, input);
2727       goto done;
2728     }
2729
2730   if (unformat (input, "%d", &table_id))
2731     ;
2732   else
2733     {
2734       error = clib_error_return (0, "expected table id `%U'",
2735                                  format_unformat_error, input);
2736       goto done;
2737     }
2738
2739   {
2740     u32 fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6,
2741                                                       table_id);
2742
2743     vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
2744     ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
2745   }
2746
2747
2748  done:
2749   return error;
2750 }
2751
2752 VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) = {
2753   .path = "set interface ip6 table",
2754   .function = add_del_ip6_interface_table,
2755   .short_help = "set interface ip6 table <intfc> <table-id>"
2756 };
2757
2758 void
2759 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t *ip,
2760                                                   u8 *mac)
2761 {
2762   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2763   /* Invert the "u" bit */
2764   ip->as_u8 [8] = mac[0] ^ (1<<1);
2765   ip->as_u8 [9] = mac[1];
2766   ip->as_u8 [10] = mac[2];
2767   ip->as_u8 [11] = 0xFF;
2768   ip->as_u8 [12] = 0xFE;
2769   ip->as_u8 [13] = mac[3];
2770   ip->as_u8 [14] = mac[4];
2771   ip->as_u8 [15] = mac[5];
2772 }
2773
2774 void
2775 ip6_ethernet_mac_address_from_link_local_address (u8 *mac,
2776                                                   ip6_address_t *ip)
2777 {
2778   /* Invert the previously inverted "u" bit */
2779   mac[0] = ip->as_u8 [8] ^ (1<<1);
2780   mac[1] = ip->as_u8 [9];
2781   mac[2] = ip->as_u8 [10];
2782   mac[3] = ip->as_u8 [13];
2783   mac[4] = ip->as_u8 [14];
2784   mac[5] = ip->as_u8 [15];
2785 }
2786
2787 static clib_error_t *
2788 test_ip6_link_command_fn (vlib_main_t * vm,
2789                           unformat_input_t * input,
2790                           vlib_cli_command_t * cmd)
2791 {
2792   u8 mac[6];
2793   ip6_address_t _a, *a = &_a;
2794
2795   if (unformat (input, "%U", unformat_ethernet_address, mac))
2796     {
2797       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2798       vlib_cli_output (vm, "Link local address: %U",
2799                        format_ip6_address, a);
2800       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2801       vlib_cli_output (vm, "Original MAC address: %U",
2802                        format_ethernet_address, mac);
2803     }
2804
2805   return 0;
2806 }
2807
2808 VLIB_CLI_COMMAND (test_link_command, static) = {
2809   .path = "test ip6 link",
2810   .function = test_ip6_link_command_fn,
2811   .short_help = "test ip6 link <mac-address>",
2812 };
2813
2814 int vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2815 {
2816   ip6_main_t * im6 = &ip6_main;
2817   ip6_fib_t * fib;
2818   uword * p = hash_get (im6->fib_index_by_table_id, table_id);
2819
2820   if (p == 0)
2821     return -1;
2822
2823   fib = ip6_fib_get (p[0]);
2824
2825   fib->flow_hash_config = flow_hash_config;
2826   return 1;
2827 }
2828
2829 static clib_error_t *
2830 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2831                               unformat_input_t * input,
2832                               vlib_cli_command_t * cmd)
2833 {
2834   int matched = 0;
2835   u32 table_id = 0;
2836   u32 flow_hash_config = 0;
2837   int rv;
2838
2839   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2840     if (unformat (input, "table %d", &table_id))
2841       matched = 1;
2842 #define _(a,v) \
2843     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2844     foreach_flow_hash_bit
2845 #undef _
2846     else break;
2847   }
2848
2849   if (matched == 0)
2850     return clib_error_return (0, "unknown input `%U'",
2851                               format_unformat_error, input);
2852
2853   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2854   switch (rv)
2855     {
2856     case 1:
2857       break;
2858
2859     case -1:
2860       return clib_error_return (0, "no such FIB table %d", table_id);
2861
2862     default:
2863       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2864       break;
2865     }
2866
2867   return 0;
2868 }
2869
2870 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
2871     .path = "set ip6 flow-hash",
2872     .short_help =
2873     "set ip table flow-hash table <fib-id> src dst sport dport proto reverse",
2874     .function = set_ip6_flow_hash_command_fn,
2875 };
2876
2877 static clib_error_t *
2878 show_ip6_local_command_fn (vlib_main_t * vm,
2879                            unformat_input_t * input,
2880                            vlib_cli_command_t * cmd)
2881 {
2882   ip6_main_t * im = &ip6_main;
2883   ip_lookup_main_t * lm = &im->lookup_main;
2884   int i;
2885
2886   vlib_cli_output (vm, "Protocols handled by ip6_local");
2887   for (i = 0; i < ARRAY_LEN(lm->local_next_by_ip_protocol); i++)
2888     {
2889       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
2890         vlib_cli_output (vm, "%d", i);
2891     }
2892   return 0;
2893 }
2894
2895
2896
2897 VLIB_CLI_COMMAND (show_ip6_local, static) = {
2898   .path = "show ip6 local",
2899   .function = show_ip6_local_command_fn,
2900   .short_help = "Show ip6 local protocol table",
2901 };
2902
2903 int vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
2904                                  u32 table_index)
2905 {
2906   vnet_main_t * vnm = vnet_get_main();
2907   vnet_interface_main_t * im = &vnm->interface_main;
2908   ip6_main_t * ipm = &ip6_main;
2909   ip_lookup_main_t * lm = &ipm->lookup_main;
2910   vnet_classify_main_t * cm = &vnet_classify_main;
2911   ip6_address_t *if_addr;
2912
2913   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
2914     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
2915
2916   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
2917     return VNET_API_ERROR_NO_SUCH_ENTRY;
2918
2919   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
2920   lm->classify_table_index_by_sw_if_index [sw_if_index] = table_index;
2921
2922   if_addr = ip6_interface_first_address (ipm, sw_if_index, NULL);
2923
2924   if (NULL != if_addr)
2925   {
2926       fib_prefix_t pfx = {
2927           .fp_len = 128,
2928           .fp_proto = FIB_PROTOCOL_IP6,
2929           .fp_addr.ip6 = *if_addr,
2930       };
2931       u32 fib_index;
2932
2933       fib_index = fib_table_get_index_for_sw_if_index(FIB_PROTOCOL_IP4,
2934                                                       sw_if_index);
2935
2936
2937       if (table_index != (u32) ~0)
2938       {
2939           dpo_id_t dpo = DPO_NULL;
2940
2941           dpo_set(&dpo,
2942                   DPO_CLASSIFY,
2943                   DPO_PROTO_IP4,
2944                   classify_dpo_create(FIB_PROTOCOL_IP4,
2945                                       table_index));
2946
2947           fib_table_entry_special_dpo_add(fib_index,
2948                                           &pfx,
2949                                           FIB_SOURCE_CLASSIFY,
2950                                           FIB_ENTRY_FLAG_NONE,
2951                                           &dpo);
2952           dpo_reset(&dpo);
2953       }
2954       else
2955       {
2956           fib_table_entry_special_remove(fib_index,
2957                                          &pfx,
2958                                          FIB_SOURCE_CLASSIFY);
2959       }
2960   }
2961
2962   return 0;
2963 }
2964
2965 static clib_error_t *
2966 set_ip6_classify_command_fn (vlib_main_t * vm,
2967                              unformat_input_t * input,
2968                              vlib_cli_command_t * cmd)
2969 {
2970   u32 table_index = ~0;
2971   int table_index_set = 0;
2972   u32 sw_if_index = ~0;
2973   int rv;
2974
2975   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
2976     if (unformat (input, "table-index %d", &table_index))
2977       table_index_set = 1;
2978     else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
2979                        vnet_get_main(), &sw_if_index))
2980         ;
2981     else
2982         break;
2983   }
2984
2985   if (table_index_set == 0)
2986       return clib_error_return (0, "classify table-index must be specified");
2987
2988   if (sw_if_index == ~0)
2989     return clib_error_return (0, "interface / subif must be specified");
2990
2991   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
2992
2993   switch (rv)
2994     {
2995     case 0:
2996       break;
2997
2998     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
2999       return clib_error_return (0, "No such interface");
3000
3001     case VNET_API_ERROR_NO_SUCH_ENTRY:
3002       return clib_error_return (0, "No such classifier table");
3003     }
3004   return 0;
3005 }
3006
3007 VLIB_CLI_COMMAND (set_ip6_classify_command, static) = {
3008     .path = "set ip6 classify",
3009     .short_help =
3010     "set ip6 classify intfc <int> table-index <index>",
3011     .function = set_ip6_classify_command_fn,
3012 };
3013
3014 static clib_error_t *
3015 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3016 {
3017   ip6_main_t * im = &ip6_main;
3018   uword heapsize = 0;
3019   u32 tmp;
3020   u32 nbuckets = 0;
3021
3022   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) {
3023     if (unformat (input, "hash-buckets %d", &tmp))
3024       nbuckets = tmp;
3025     else if (unformat (input, "heap-size %dm", &tmp))
3026       heapsize = ((u64)tmp) << 20;
3027     else if (unformat (input, "heap-size %dM", &tmp))
3028       heapsize = ((u64)tmp) << 20;
3029     else if (unformat (input, "heap-size %dg", &tmp))
3030       heapsize = ((u64)tmp) << 30;
3031     else if (unformat (input, "heap-size %dG", &tmp))
3032       heapsize = ((u64)tmp) << 30;
3033     else
3034       return clib_error_return (0, "unknown input '%U'",
3035                                 format_unformat_error, input);
3036   }
3037
3038   im->lookup_table_nbuckets = nbuckets;
3039   im->lookup_table_size = heapsize;
3040
3041   return 0;
3042 }
3043
3044 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3045
3046 #define TEST_CODE 1
3047 #if TEST_CODE > 0
3048
3049 static clib_error_t *
3050 set_interface_ip6_output_feature_command_fn (vlib_main_t * vm,
3051                                              unformat_input_t * input,
3052                                              vlib_cli_command_t * cmd)
3053 {
3054   vnet_main_t * vnm = vnet_get_main();
3055   u32 sw_if_index = ~0;
3056   int is_add = 1;
3057   ip6_main_t * im = &ip6_main;
3058   ip_lookup_main_t * lm = &im->lookup_main;
3059
3060   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3061     {
3062       if (unformat (input, "%U", unformat_vnet_sw_interface, vnm, &sw_if_index))
3063         ;
3064       else if (unformat (input, "del"))
3065         is_add = 0;
3066       else
3067         break;
3068     }
3069
3070   if (sw_if_index == ~0)
3071     return clib_error_return (0, "unknown interface `%U'",
3072                               format_unformat_error, input);
3073
3074   lm->tx_sw_if_has_ip_output_features =
3075     clib_bitmap_set (lm->tx_sw_if_has_ip_output_features, sw_if_index, is_add);
3076
3077   return 0;
3078 }
3079
3080 VLIB_CLI_COMMAND (set_interface_ip6_output_feature, static) = {
3081   .path = "set interface ip6 output feature",
3082   .function = set_interface_ip6_output_feature_command_fn,
3083   .short_help = "set interface output feature <intfc>",
3084 };
3085 #endif /* TEST_CODE */