src/vnet/ip/ip6_forward.c

   1 /*
   2  * Copyright (c) 2016 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * ip/ip6_forward.c: IP v6 forwarding
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #include <vnet/vnet.h>
  41 #include <vnet/ip/ip.h>
  42 #include <vnet/ip/ip6_neighbor.h>
  43 #include <vnet/ethernet/ethernet.h>     /* for ethernet_header_t */
  44 #include <vnet/srp/srp.h>       /* for srp_hw_interface_class */
  45 #include <vppinfra/cache.h>
  46 #include <vnet/fib/fib_urpf_list.h>     /* for FIB uRPF check */
  47 #include <vnet/fib/ip6_fib.h>
  48 #include <vnet/mfib/ip6_mfib.h>
  49 #include <vnet/dpo/load_balance_map.h>
  50 #include <vnet/dpo/classify_dpo.h>
  51
  52 #include <vppinfra/bihash_template.c>
  53 #include <vnet/ip/ip6_forward.h>
  54
  55 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
  56 #define OI_DECAP   0x80000000
  57
  58 static void
  59 ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
  60                           ip6_main_t * im, u32 fib_index,
  61                           ip_interface_address_t * a)
  62 {
  63   ip_lookup_main_t *lm = &im->lookup_main;
  64   ip6_address_t *address = ip_interface_address_get_address (lm, a);
  65   fib_prefix_t pfx = {
  66     .fp_len = a->address_length,
  67     .fp_proto = FIB_PROTOCOL_IP6,
  68     .fp_addr.ip6 = *address,
  69   };
  70
  71   if (a->address_length < 128)
  72     {
  73       fib_table_entry_update_one_path (fib_index,
  74                                        &pfx,
  75                                        FIB_SOURCE_INTERFACE,
  76                                        (FIB_ENTRY_FLAG_CONNECTED |
  77                                         FIB_ENTRY_FLAG_ATTACHED),
  78                                        DPO_PROTO_IP6,
  79                                        /* No next-hop address */
  80                                        NULL, sw_if_index,
  81                                        /* invalid FIB index */
  82                                        ~0, 1,
  83                                        /* no label stack */
  84                                        NULL, FIB_ROUTE_PATH_FLAG_NONE);
  85     }
  86
  87   pfx.fp_len = 128;
  88   if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
  89     {
  90       u32 classify_table_index =
  91         lm->classify_table_index_by_sw_if_index[sw_if_index];
  92       if (classify_table_index != (u32) ~ 0)
  93         {
  94           dpo_id_t dpo = DPO_INVALID;
  95
  96           dpo_set (&dpo,
  97                    DPO_CLASSIFY,
  98                    DPO_PROTO_IP6,
  99                    classify_dpo_create (DPO_PROTO_IP6, classify_table_index));
 100
 101           fib_table_entry_special_dpo_add (fib_index,
 102                                            &pfx,
 103                                            FIB_SOURCE_CLASSIFY,
 104                                            FIB_ENTRY_FLAG_NONE, &dpo);
 105           dpo_reset (&dpo);
 106         }
 107     }
 108
 109   fib_table_entry_update_one_path (fib_index, &pfx,
 110                                    FIB_SOURCE_INTERFACE,
 111                                    (FIB_ENTRY_FLAG_CONNECTED |
 112                                     FIB_ENTRY_FLAG_LOCAL),
 113                                    DPO_PROTO_IP6,
 114                                    &pfx.fp_addr,
 115                                    sw_if_index, ~0,
 116                                    1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 117 }
 118
 119 static void
 120 ip6_del_interface_routes (ip6_main_t * im,
 121                           u32 fib_index,
 122                           ip6_address_t * address, u32 address_length)
 123 {
 124   fib_prefix_t pfx = {
 125     .fp_len = address_length,
 126     .fp_proto = FIB_PROTOCOL_IP6,
 127     .fp_addr.ip6 = *address,
 128   };
 129
 130   if (pfx.fp_len < 128)
 131     {
 132       fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 133
 134     }
 135
 136   pfx.fp_len = 128;
 137   fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
 138 }
 139
 140 void
 141 ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 142 {
 143   ip6_main_t *im = &ip6_main;
 144
 145   vec_validate_init_empty (im->ip_enabled_by_sw_if_index, sw_if_index, 0);
 146
 147   /*
 148    * enable/disable only on the 1<->0 transition
 149    */
 150   if (is_enable)
 151     {
 152       if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
 153         return;
 154     }
 155   else
 156     {
 157       /* The ref count is 0 when an address is removed from an interface that has
 158        * no address - this is not a ciritical error */
 159       if (0 == im->ip_enabled_by_sw_if_index[sw_if_index] ||
 160           0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
 161         return;
 162     }
 163
 164   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 165                                !is_enable, 0, 0);
 166
 167   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 168                                sw_if_index, !is_enable, 0, 0);
 169 }
 170
 171 /* get first interface address */
 172 ip6_address_t *
 173 ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
 174 {
 175   ip_lookup_main_t *lm = &im->lookup_main;
 176   ip_interface_address_t *ia = 0;
 177   ip6_address_t *result = 0;
 178
 179   /* *INDENT-OFF* */
 180   foreach_ip_interface_address (lm, ia, sw_if_index,
 181                                 1 /* honor unnumbered */,
 182   ({
 183     ip6_address_t * a = ip_interface_address_get_address (lm, ia);
 184     result = a;
 185     break;
 186   }));
 187   /* *INDENT-ON* */
 188   return result;
 189 }
 190
 191 clib_error_t *
 192 ip6_add_del_interface_address (vlib_main_t * vm,
 193                                u32 sw_if_index,
 194                                ip6_address_t * address,
 195                                u32 address_length, u32 is_del)
 196 {
 197   vnet_main_t *vnm = vnet_get_main ();
 198   ip6_main_t *im = &ip6_main;
 199   ip_lookup_main_t *lm = &im->lookup_main;
 200   clib_error_t *error;
 201   u32 if_address_index;
 202   ip6_address_fib_t ip6_af, *addr_fib = 0;
 203
 204   /* local0 interface doesn't support IP addressing */
 205   if (sw_if_index == 0)
 206     {
 207       return
 208         clib_error_create ("local0 interface doesn't support IP addressing");
 209     }
 210
 211   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 212   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 213
 214   ip6_addr_fib_init (&ip6_af, address,
 215                      vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
 216   vec_add1 (addr_fib, ip6_af);
 217
 218   {
 219     uword elts_before = pool_elts (lm->if_address_pool);
 220
 221     error = ip_interface_address_add_del
 222       (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
 223     if (error)
 224       goto done;
 225
 226     /* Pool did not grow: add duplicate address. */
 227     if (elts_before == pool_elts (lm->if_address_pool))
 228       goto done;
 229   }
 230
 231   ip6_sw_interface_enable_disable (sw_if_index, !is_del);
 232
 233   if (is_del)
 234     ip6_del_interface_routes (im, ip6_af.fib_index, address, address_length);
 235   else
 236     ip6_add_interface_routes (vnm, sw_if_index,
 237                               im, ip6_af.fib_index,
 238                               pool_elt_at_index (lm->if_address_pool,
 239                                                  if_address_index));
 240
 241   {
 242     ip6_add_del_interface_address_callback_t *cb;
 243     vec_foreach (cb, im->add_del_interface_address_callbacks)
 244       cb->function (im, cb->function_opaque, sw_if_index,
 245                     address, address_length, if_address_index, is_del);
 246   }
 247
 248 done:
 249   vec_free (addr_fib);
 250   return error;
 251 }
 252
 253 clib_error_t *
 254 ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 255 {
 256   ip6_main_t *im = &ip6_main;
 257   ip_interface_address_t *ia;
 258   ip6_address_t *a;
 259   u32 is_admin_up, fib_index;
 260
 261   /* Fill in lookup tables with default table (0). */
 262   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 263
 264   vec_validate_init_empty (im->
 265                            lookup_main.if_address_pool_index_by_sw_if_index,
 266                            sw_if_index, ~0);
 267
 268   is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
 269
 270   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 271
 272   /* *INDENT-OFF* */
 273   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
 274                                 0 /* honor unnumbered */,
 275   ({
 276     a = ip_interface_address_get_address (&im->lookup_main, ia);
 277     if (is_admin_up)
 278       ip6_add_interface_routes (vnm, sw_if_index,
 279                                 im, fib_index,
 280                                 ia);
 281     else
 282       ip6_del_interface_routes (im, fib_index,
 283                                 a, ia->address_length);
 284   }));
 285   /* *INDENT-ON* */
 286
 287   return 0;
 288 }
 289
 290 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 291
 292 /* Built-in ip6 unicast rx feature path definition */
 293 /* *INDENT-OFF* */
 294 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 295 {
 296   .arc_name  = "ip6-unicast",
 297   .start_nodes = VNET_FEATURES ("ip6-input"),
 298   .arc_index_ptr = &ip6_main.lookup_main.ucast_feature_arc_index,
 299 };
 300
 301 VNET_FEATURE_INIT (ip6_flow_classify, static) =
 302 {
 303   .arc_name = "ip6-unicast",
 304   .node_name = "ip6-flow-classify",
 305   .runs_before = VNET_FEATURES ("ip6-inacl"),
 306 };
 307
 308 VNET_FEATURE_INIT (ip6_inacl, static) =
 309 {
 310   .arc_name = "ip6-unicast",
 311   .node_name = "ip6-inacl",
 312   .runs_before = VNET_FEATURES ("ip6-policer-classify"),
 313 };
 314
 315 VNET_FEATURE_INIT (ip6_policer_classify, static) =
 316 {
 317   .arc_name = "ip6-unicast",
 318   .node_name = "ip6-policer-classify",
 319   .runs_before = VNET_FEATURES ("ipsec-input-ip6"),
 320 };
 321
 322 VNET_FEATURE_INIT (ip6_ipsec, static) =
 323 {
 324   .arc_name = "ip6-unicast",
 325   .node_name = "ipsec-input-ip6",
 326   .runs_before = VNET_FEATURES ("l2tp-decap"),
 327 };
 328
 329 VNET_FEATURE_INIT (ip6_l2tp, static) =
 330 {
 331   .arc_name = "ip6-unicast",
 332   .node_name = "l2tp-decap",
 333   .runs_before = VNET_FEATURES ("vpath-input-ip6"),
 334 };
 335
 336 VNET_FEATURE_INIT (ip6_vpath, static) =
 337 {
 338   .arc_name = "ip6-unicast",
 339   .node_name = "vpath-input-ip6",
 340   .runs_before = VNET_FEATURES ("ip6-vxlan-bypass"),
 341 };
 342
 343 VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
 344 {
 345   .arc_name = "ip6-unicast",
 346   .node_name = "ip6-vxlan-bypass",
 347   .runs_before = VNET_FEATURES ("ip6-lookup"),
 348 };
 349
 350 VNET_FEATURE_INIT (ip6_not_enabled, static) =
 351 {
 352   .arc_name = "ip6-unicast",
 353   .node_name = "ip6-not-enabled",
 354   .runs_before = VNET_FEATURES ("ip6-lookup"),
 355 };
 356
 357 VNET_FEATURE_INIT (ip6_lookup, static) =
 358 {
 359   .arc_name = "ip6-unicast",
 360   .node_name = "ip6-lookup",
 361   .runs_before = 0,  /*last feature*/
 362 };
 363
 364 /* Built-in ip6 multicast rx feature path definition (none now) */
 365 VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 366 {
 367   .arc_name  = "ip6-multicast",
 368   .start_nodes = VNET_FEATURES ("ip6-input"),
 369   .arc_index_ptr = &ip6_main.lookup_main.mcast_feature_arc_index,
 370 };
 371
 372 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
 373   .arc_name = "ip6-multicast",
 374   .node_name = "vpath-input-ip6",
 375   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 376 };
 377
 378 VNET_FEATURE_INIT (ip6_not_enabled_mc, static) = {
 379   .arc_name = "ip6-multicast",
 380   .node_name = "ip6-not-enabled",
 381   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 382 };
 383
 384 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
 385   .arc_name = "ip6-multicast",
 386   .node_name = "ip6-mfib-forward-lookup",
 387   .runs_before = 0, /* last feature */
 388 };
 389
 390 /* Built-in ip4 tx feature path definition */
 391 VNET_FEATURE_ARC_INIT (ip6_output, static) =
 392 {
 393   .arc_name  = "ip6-output",
 394   .start_nodes = VNET_FEATURES ("ip6-rewrite", "ip6-midchain", "ip6-dvr-dpo"),
 395   .arc_index_ptr = &ip6_main.lookup_main.output_feature_arc_index,
 396 };
 397
 398 VNET_FEATURE_INIT (ip6_outacl, static) = {
 399   .arc_name = "ip6-output",
 400   .node_name = "ip6-outacl",
 401   .runs_before = VNET_FEATURES ("ipsec-output-ip6"),
 402 };
 403
 404 VNET_FEATURE_INIT (ip6_ipsec_output, static) = {
 405   .arc_name = "ip6-output",
 406   .node_name = "ipsec-output-ip6",
 407   .runs_before = VNET_FEATURES ("interface-output"),
 408 };
 409
 410 VNET_FEATURE_INIT (ip6_interface_output, static) = {
 411   .arc_name = "ip6-output",
 412   .node_name = "interface-output",
 413   .runs_before = 0, /* not before any other features */
 414 };
 415 /* *INDENT-ON* */
 416
 417 clib_error_t *
 418 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 419 {
 420   ip6_main_t *im = &ip6_main;
 421
 422   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
 423   vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 424
 425   if (!is_add)
 426     {
 427       /* Ensure that IPv6 is disabled */
 428       ip6_main_t *im6 = &ip6_main;
 429       ip_lookup_main_t *lm6 = &im6->lookup_main;
 430       ip_interface_address_t *ia = 0;
 431       ip6_address_t *address;
 432       vlib_main_t *vm = vlib_get_main ();
 433
 434       ip6_neighbor_sw_interface_add_del (vnm, sw_if_index, 0 /* is_add */ );
 435       /* *INDENT-OFF* */
 436       foreach_ip_interface_address (lm6, ia, sw_if_index, 1 /* honor unnumbered */,
 437       ({
 438         address = ip_interface_address_get_address (lm6, ia);
 439         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
 440       }));
 441       /* *INDENT-ON* */
 442       ip6_mfib_interface_enable_disable (sw_if_index, 0);
 443     }
 444
 445   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
 446                                is_add, 0, 0);
 447
 448   vnet_feature_enable_disable ("ip6-multicast", "ip6-not-enabled",
 449                                sw_if_index, is_add, 0, 0);
 450
 451   return /* no error */ 0;
 452 }
 453
 454 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip6_sw_interface_add_del);
 455
 456 static uword
 457 ip6_lookup (vlib_main_t * vm,
 458             vlib_node_runtime_t * node, vlib_frame_t * frame)
 459 {
 460   return ip6_lookup_inline (vm, node, frame);
 461 }
 462
 463 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 464
 465 /* *INDENT-OFF* */
 466 VLIB_REGISTER_NODE (ip6_lookup_node) =
 467 {
 468   .function = ip6_lookup,
 469   .name = "ip6-lookup",
 470   .vector_size = sizeof (u32),
 471   .format_trace = format_ip6_lookup_trace,
 472   .n_next_nodes = IP6_LOOKUP_N_NEXT,
 473   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 474 };
 475 /* *INDENT-ON* */
 476
 477 VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup);
 478
 479 always_inline uword
 480 ip6_load_balance (vlib_main_t * vm,
 481                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 482 {
 483   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
 484   u32 n_left_from, n_left_to_next, *from, *to_next;
 485   ip_lookup_next_t next;
 486   u32 thread_index = vlib_get_thread_index ();
 487   ip6_main_t *im = &ip6_main;
 488
 489   from = vlib_frame_vector_args (frame);
 490   n_left_from = frame->n_vectors;
 491   next = node->cached_next_index;
 492
 493   if (node->flags & VLIB_NODE_FLAG_TRACE)
 494     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
 495
 496   while (n_left_from > 0)
 497     {
 498       vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
 499
 500
 501       while (n_left_from >= 4 && n_left_to_next >= 2)
 502         {
 503           ip_lookup_next_t next0, next1;
 504           const load_balance_t *lb0, *lb1;
 505           vlib_buffer_t *p0, *p1;
 506           u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
 507           const ip6_header_t *ip0, *ip1;
 508           const dpo_id_t *dpo0, *dpo1;
 509
 510           /* Prefetch next iteration. */
 511           {
 512             vlib_buffer_t *p2, *p3;
 513
 514             p2 = vlib_get_buffer (vm, from[2]);
 515             p3 = vlib_get_buffer (vm, from[3]);
 516
 517             vlib_prefetch_buffer_header (p2, STORE);
 518             vlib_prefetch_buffer_header (p3, STORE);
 519
 520             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
 521             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
 522           }
 523
 524           pi0 = to_next[0] = from[0];
 525           pi1 = to_next[1] = from[1];
 526
 527           from += 2;
 528           n_left_from -= 2;
 529           to_next += 2;
 530           n_left_to_next -= 2;
 531
 532           p0 = vlib_get_buffer (vm, pi0);
 533           p1 = vlib_get_buffer (vm, pi1);
 534
 535           ip0 = vlib_buffer_get_current (p0);
 536           ip1 = vlib_buffer_get_current (p1);
 537           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 538           lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 539
 540           lb0 = load_balance_get (lbi0);
 541           lb1 = load_balance_get (lbi1);
 542
 543           /*
 544            * this node is for via FIBs we can re-use the hash value from the
 545            * to node if present.
 546            * We don't want to use the same hash value at each level in the recursion
 547            * graph as that would lead to polarisation
 548            */
 549           hc0 = hc1 = 0;
 550
 551           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 552             {
 553               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 554                 {
 555                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 556                     vnet_buffer (p0)->ip.flow_hash >> 1;
 557                 }
 558               else
 559                 {
 560                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 561                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 562                 }
 563               dpo0 =
 564                 load_balance_get_fwd_bucket (lb0,
 565                                              (hc0 &
 566                                               lb0->lb_n_buckets_minus_1));
 567             }
 568           else
 569             {
 570               dpo0 = load_balance_get_bucket_i (lb0, 0);
 571             }
 572           if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
 573             {
 574               if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
 575                 {
 576                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 577                     vnet_buffer (p1)->ip.flow_hash >> 1;
 578                 }
 579               else
 580                 {
 581                   hc1 = vnet_buffer (p1)->ip.flow_hash =
 582                     ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
 583                 }
 584               dpo1 =
 585                 load_balance_get_fwd_bucket (lb1,
 586                                              (hc1 &
 587                                               lb1->lb_n_buckets_minus_1));
 588             }
 589           else
 590             {
 591               dpo1 = load_balance_get_bucket_i (lb1, 0);
 592             }
 593
 594           next0 = dpo0->dpoi_next_node;
 595           next1 = dpo1->dpoi_next_node;
 596
 597           /* Only process the HBH Option Header if explicitly configured to do so */
 598           if (PREDICT_FALSE
 599               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 600             {
 601               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 602                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 603             }
 604           /* Only process the HBH Option Header if explicitly configured to do so */
 605           if (PREDICT_FALSE
 606               (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 607             {
 608               next1 = (dpo_is_adj (dpo1) && im->hbh_enabled) ?
 609                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next1;
 610             }
 611
 612           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 613           vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 614
 615           vlib_increment_combined_counter
 616             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 617           vlib_increment_combined_counter
 618             (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 619
 620           vlib_validate_buffer_enqueue_x2 (vm, node, next,
 621                                            to_next, n_left_to_next,
 622                                            pi0, pi1, next0, next1);
 623         }
 624
 625       while (n_left_from > 0 && n_left_to_next > 0)
 626         {
 627           ip_lookup_next_t next0;
 628           const load_balance_t *lb0;
 629           vlib_buffer_t *p0;
 630           u32 pi0, lbi0, hc0;
 631           const ip6_header_t *ip0;
 632           const dpo_id_t *dpo0;
 633
 634           pi0 = from[0];
 635           to_next[0] = pi0;
 636           from += 1;
 637           to_next += 1;
 638           n_left_to_next -= 1;
 639           n_left_from -= 1;
 640
 641           p0 = vlib_get_buffer (vm, pi0);
 642
 643           ip0 = vlib_buffer_get_current (p0);
 644           lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 645
 646           lb0 = load_balance_get (lbi0);
 647
 648           hc0 = 0;
 649           if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
 650             {
 651               if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
 652                 {
 653                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 654                     vnet_buffer (p0)->ip.flow_hash >> 1;
 655                 }
 656               else
 657                 {
 658                   hc0 = vnet_buffer (p0)->ip.flow_hash =
 659                     ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
 660                 }
 661               dpo0 =
 662                 load_balance_get_fwd_bucket (lb0,
 663                                              (hc0 &
 664                                               lb0->lb_n_buckets_minus_1));
 665             }
 666           else
 667             {
 668               dpo0 = load_balance_get_bucket_i (lb0, 0);
 669             }
 670
 671           next0 = dpo0->dpoi_next_node;
 672           vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 673
 674           /* Only process the HBH Option Header if explicitly configured to do so */
 675           if (PREDICT_FALSE
 676               (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 677             {
 678               next0 = (dpo_is_adj (dpo0) && im->hbh_enabled) ?
 679                 (ip_lookup_next_t) IP6_LOOKUP_NEXT_HOP_BY_HOP : next0;
 680             }
 681
 682           vlib_increment_combined_counter
 683             (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 684
 685           vlib_validate_buffer_enqueue_x1 (vm, node, next,
 686                                            to_next, n_left_to_next,
 687                                            pi0, next0);
 688         }
 689
 690       vlib_put_next_frame (vm, node, next, n_left_to_next);
 691     }
 692
 693   return frame->n_vectors;
 694 }
 695
 696 /* *INDENT-OFF* */
 697 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 698 {
 699   .function = ip6_load_balance,
 700   .name = "ip6-load-balance",
 701   .vector_size = sizeof (u32),
 702   .sibling_of = "ip6-lookup",
 703   .format_trace = format_ip6_lookup_trace,
 704 };
 705 /* *INDENT-ON* */
 706
 707 VLIB_NODE_FUNCTION_MULTIARCH (ip6_load_balance_node, ip6_load_balance);
 708
 709 typedef struct
 710 {
 711   /* Adjacency taken. */
 712   u32 adj_index;
 713   u32 flow_hash;
 714   u32 fib_index;
 715
 716   /* Packet data, possibly *after* rewrite. */
 717   u8 packet_data[128 - 1 * sizeof (u32)];
 718 }
 719 ip6_forward_next_trace_t;
 720
 721 u8 *
 722 format_ip6_forward_next_trace (u8 * s, va_list * args)
 723 {
 724   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 725   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 726   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 727   u32 indent = format_get_indent (s);
 728
 729   s = format (s, "%U%U",
 730               format_white_space, indent,
 731               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 732   return s;
 733 }
 734
 735 static u8 *
 736 format_ip6_lookup_trace (u8 * s, va_list * args)
 737 {
 738   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 739   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 740   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 741   u32 indent = format_get_indent (s);
 742
 743   s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
 744               t->fib_index, t->adj_index, t->flow_hash);
 745   s = format (s, "\n%U%U",
 746               format_white_space, indent,
 747               format_ip6_header, t->packet_data, sizeof (t->packet_data));
 748   return s;
 749 }
 750
 751
 752 static u8 *
 753 format_ip6_rewrite_trace (u8 * s, va_list * args)
 754 {
 755   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 756   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 757   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
 758   u32 indent = format_get_indent (s);
 759
 760   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
 761               t->fib_index, t->adj_index, format_ip_adjacency,
 762               t->adj_index, FORMAT_IP_ADJACENCY_NONE, t->flow_hash);
 763   s = format (s, "\n%U%U",
 764               format_white_space, indent,
 765               format_ip_adjacency_packet_data,
 766               t->adj_index, t->packet_data, sizeof (t->packet_data));
 767   return s;
 768 }
 769
 770 /* Common trace function for all ip6-forward next nodes. */
 771 void
 772 ip6_forward_next_trace (vlib_main_t * vm,
 773                         vlib_node_runtime_t * node,
 774                         vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
 775 {
 776   u32 *from, n_left;
 777   ip6_main_t *im = &ip6_main;
 778
 779   n_left = frame->n_vectors;
 780   from = vlib_frame_vector_args (frame);
 781
 782   while (n_left >= 4)
 783     {
 784       u32 bi0, bi1;
 785       vlib_buffer_t *b0, *b1;
 786       ip6_forward_next_trace_t *t0, *t1;
 787
 788       /* Prefetch next iteration. */
 789       vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
 790       vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
 791
 792       bi0 = from[0];
 793       bi1 = from[1];
 794
 795       b0 = vlib_get_buffer (vm, bi0);
 796       b1 = vlib_get_buffer (vm, bi1);
 797
 798       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 799         {
 800           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 801           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 802           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 803           t0->fib_index =
 804             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 805              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 806             vec_elt (im->fib_index_by_sw_if_index,
 807                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 808
 809           clib_memcpy (t0->packet_data,
 810                        vlib_buffer_get_current (b0),
 811                        sizeof (t0->packet_data));
 812         }
 813       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 814         {
 815           t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 816           t1->adj_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
 817           t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
 818           t1->fib_index =
 819             (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
 820              (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
 821             vec_elt (im->fib_index_by_sw_if_index,
 822                      vnet_buffer (b1)->sw_if_index[VLIB_RX]);
 823
 824           clib_memcpy (t1->packet_data,
 825                        vlib_buffer_get_current (b1),
 826                        sizeof (t1->packet_data));
 827         }
 828       from += 2;
 829       n_left -= 2;
 830     }
 831
 832   while (n_left >= 1)
 833     {
 834       u32 bi0;
 835       vlib_buffer_t *b0;
 836       ip6_forward_next_trace_t *t0;
 837
 838       bi0 = from[0];
 839
 840       b0 = vlib_get_buffer (vm, bi0);
 841
 842       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 843         {
 844           t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 845           t0->adj_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
 846           t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
 847           t0->fib_index =
 848             (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
 849              (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
 850             vec_elt (im->fib_index_by_sw_if_index,
 851                      vnet_buffer (b0)->sw_if_index[VLIB_RX]);
 852
 853           clib_memcpy (t0->packet_data,
 854                        vlib_buffer_get_current (b0),
 855                        sizeof (t0->packet_data));
 856         }
 857       from += 1;
 858       n_left -= 1;
 859     }
 860 }
 861
 862 /* Compute TCP/UDP/ICMP6 checksum in software. */
 863 u16
 864 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
 865                                    ip6_header_t * ip0, int *bogus_lengthp)
 866 {
 867   ip_csum_t sum0;
 868   u16 sum16, payload_length_host_byte_order;
 869   u32 i, n_this_buffer, n_bytes_left;
 870   u32 headers_size = sizeof (ip0[0]);
 871   void *data_this_buffer;
 872
 873   ASSERT (bogus_lengthp);
 874   *bogus_lengthp = 0;
 875
 876   /* Initialize checksum with ip header. */
 877   sum0 = ip0->payload_length + clib_host_to_net_u16 (ip0->protocol);
 878   payload_length_host_byte_order = clib_net_to_host_u16 (ip0->payload_length);
 879   data_this_buffer = (void *) (ip0 + 1);
 880
 881   for (i = 0; i < ARRAY_LEN (ip0->src_address.as_uword); i++)
 882     {
 883       sum0 = ip_csum_with_carry (sum0,
 884                                  clib_mem_unaligned (&ip0->
 885                                                      src_address.as_uword[i],
 886                                                      uword));
 887       sum0 =
 888         ip_csum_with_carry (sum0,
 889                             clib_mem_unaligned (&ip0->dst_address.as_uword[i],
 890                                                 uword));
 891     }
 892
 893   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
 894    * or UDP-Ping packets */
 895   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
 896     {
 897       u32 skip_bytes;
 898       ip6_hop_by_hop_ext_t *ext_hdr =
 899         (ip6_hop_by_hop_ext_t *) data_this_buffer;
 900
 901       /* validate really icmp6 next */
 902       ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
 903               || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
 904
 905       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
 906       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
 907
 908       payload_length_host_byte_order -= skip_bytes;
 909       headers_size += skip_bytes;
 910     }
 911
 912   n_bytes_left = n_this_buffer = payload_length_host_byte_order;
 913   if (p0 && n_this_buffer + headers_size > p0->current_length)
 914     n_this_buffer =
 915       p0->current_length >
 916       headers_size ? p0->current_length - headers_size : 0;
 917   while (1)
 918     {
 919       sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
 920       n_bytes_left -= n_this_buffer;
 921       if (n_bytes_left == 0)
 922         break;
 923
 924       if (!(p0->flags & VLIB_BUFFER_NEXT_PRESENT))
 925         {
 926           *bogus_lengthp = 1;
 927           return 0xfefe;
 928         }
 929       p0 = vlib_get_buffer (vm, p0->next_buffer);
 930       data_this_buffer = vlib_buffer_get_current (p0);
 931       n_this_buffer = p0->current_length;
 932     }
 933
 934   sum16 = ~ip_csum_fold (sum0);
 935
 936   return sum16;
 937 }
 938
 939 u32
 940 ip6_tcp_udp_icmp_validate_checksum (vlib_main_t * vm, vlib_buffer_t * p0)
 941 {
 942   ip6_header_t *ip0 = vlib_buffer_get_current (p0);
 943   udp_header_t *udp0;
 944   u16 sum16;
 945   int bogus_length;
 946
 947   /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
 948   ASSERT (ip0->protocol == IP_PROTOCOL_TCP
 949           || ip0->protocol == IP_PROTOCOL_ICMP6
 950           || ip0->protocol == IP_PROTOCOL_UDP
 951           || ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS);
 952
 953   udp0 = (void *) (ip0 + 1);
 954   if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
 955     {
 956       p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
 957                     | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
 958       return p0->flags;
 959     }
 960
 961   sum16 = ip6_tcp_udp_icmp_compute_checksum (vm, p0, ip0, &bogus_length);
 962
 963   p0->flags |= (VNET_BUFFER_F_L4_CHECKSUM_COMPUTED
 964                 | ((sum16 == 0) << VNET_BUFFER_F_LOG2_L4_CHECKSUM_CORRECT));
 965
 966   return p0->flags;
 967 }
 968
 969 /**
 970  * @brief returns number of links on which src is reachable.
 971  */
 972 always_inline int
 973 ip6_urpf_loose_check (ip6_main_t * im, vlib_buffer_t * b, ip6_header_t * i)
 974 {
 975   const load_balance_t *lb0;
 976   index_t lbi;
 977
 978   lbi = ip6_fib_table_fwding_lookup_with_if_index (im,
 979                                                    vnet_buffer
 980                                                    (b)->sw_if_index[VLIB_RX],
 981                                                    &i->src_address);
 982
 983   lb0 = load_balance_get (lbi);
 984
 985   return (fib_urpf_check_size (lb0->lb_urpf));
 986 }
 987
 988 always_inline u8
 989 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
 990                            u32 * udp_offset0)
 991 {
 992   u32 proto0;
 993   proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
 994   if (proto0 != IP_PROTOCOL_UDP)
 995     {
 996       proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
 997       proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
 998     }
 999   return proto0;
1000 }
1001
1002 /* *INDENT-OFF* */
1003 VNET_FEATURE_ARC_INIT (ip6_local) =
1004 {
1005   .arc_name  = "ip6-local",
1006   .start_nodes = VNET_FEATURES ("ip6-local"),
1007 };
1008 /* *INDENT-ON* */
1009
1010 static uword
1011 ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
1012                   vlib_frame_t * frame, int head_of_feature_arc)
1013 {
1014   ip6_main_t *im = &ip6_main;
1015   ip_lookup_main_t *lm = &im->lookup_main;
1016   ip_local_next_t next_index;
1017   u32 *from, *to_next, n_left_from, n_left_to_next;
1018   vlib_node_runtime_t *error_node =
1019     vlib_node_get_runtime (vm, ip6_input_node.index);
1020   u8 arc_index = vnet_feat_arc_ip6_local.feature_arc_index;
1021
1022   from = vlib_frame_vector_args (frame);
1023   n_left_from = frame->n_vectors;
1024   next_index = node->cached_next_index;
1025
1026   if (node->flags & VLIB_NODE_FLAG_TRACE)
1027     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1028
1029   while (n_left_from > 0)
1030     {
1031       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1032
1033       while (n_left_from >= 4 && n_left_to_next >= 2)
1034         {
1035           vlib_buffer_t *p0, *p1;
1036           ip6_header_t *ip0, *ip1;
1037           udp_header_t *udp0, *udp1;
1038           u32 pi0, ip_len0, udp_len0, flags0, next0;
1039           u32 pi1, ip_len1, udp_len1, flags1, next1;
1040           i32 len_diff0, len_diff1;
1041           u8 error0, type0, good_l4_csum0, is_tcp_udp0;
1042           u8 error1, type1, good_l4_csum1, is_tcp_udp1;
1043           u32 udp_offset0, udp_offset1;
1044
1045           pi0 = to_next[0] = from[0];
1046           pi1 = to_next[1] = from[1];
1047           from += 2;
1048           n_left_from -= 2;
1049           to_next += 2;
1050           n_left_to_next -= 2;
1051
1052           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1053
1054           p0 = vlib_get_buffer (vm, pi0);
1055           p1 = vlib_get_buffer (vm, pi1);
1056
1057           ip0 = vlib_buffer_get_current (p0);
1058           ip1 = vlib_buffer_get_current (p1);
1059
1060           if (head_of_feature_arc == 0)
1061             goto skip_checks;
1062
1063           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1064           vnet_buffer (p1)->l3_hdr_offset = p1->current_data;
1065
1066           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1067           type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol];
1068
1069           flags0 = p0->flags;
1070           flags1 = p1->flags;
1071
1072           is_tcp_udp0 = ip6_next_proto_is_tcp_udp (p0, ip0, &udp_offset0);
1073           is_tcp_udp1 = ip6_next_proto_is_tcp_udp (p1, ip1, &udp_offset1);
1074
1075           good_l4_csum0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1076                            || (flags0 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1077                                || flags0 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1078             != 0;
1079           good_l4_csum1 = (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1080                            || (flags1 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1081                                || flags1 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1082             != 0;
1083           len_diff0 = 0;
1084           len_diff1 = 0;
1085
1086           if (PREDICT_TRUE (is_tcp_udp0))
1087             {
1088               udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0);
1089               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1090               good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP
1091                 && udp0->checksum == 0;
1092               /* Verify UDP length. */
1093               if (is_tcp_udp0 == IP_PROTOCOL_UDP)
1094                 {
1095                   ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1096                   udp_len0 = clib_net_to_host_u16 (udp0->length);
1097                   len_diff0 = ip_len0 - udp_len0;
1098                 }
1099             }
1100           if (PREDICT_TRUE (is_tcp_udp1))
1101             {
1102               udp1 = (udp_header_t *) ((u8 *) ip1 + udp_offset1);
1103               /* Don't verify UDP checksum for packets with explicit zero checksum. */
1104               good_l4_csum1 |= type1 == IP_BUILTIN_PROTOCOL_UDP
1105                 && udp1->checksum == 0;
1106               /* Verify UDP length. */
1107               if (is_tcp_udp1 == IP_PROTOCOL_UDP)
1108                 {
1109                   ip_len1 = clib_net_to_host_u16 (ip1->payload_length);
1110                   udp_len1 = clib_net_to_host_u16 (udp1->length);
1111                   len_diff1 = ip_len1 - udp_len1;
1112                 }
1113             }
1114
1115           good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1116           good_l4_csum1 |= type1 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1117
1118           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1119           len_diff1 = type1 == IP_BUILTIN_PROTOCOL_UDP ? len_diff1 : 0;
1120
1121           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1122                              && !good_l4_csum0
1123                              && !(flags0 &
1124                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1125             {
1126               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1127               good_l4_csum0 =
1128                 (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1129             }
1130           if (PREDICT_FALSE (type1 != IP_BUILTIN_PROTOCOL_UNKNOWN
1131                              && !good_l4_csum1
1132                              && !(flags1 &
1133                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1134             {
1135               flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, p1);
1136               good_l4_csum1 =
1137                 (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1138             }
1139
1140           error0 = error1 = IP6_ERROR_UNKNOWN_PROTOCOL;
1141           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1142           error1 = len_diff1 < 0 ? IP6_ERROR_UDP_LENGTH : error1;
1143
1144           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1145                   IP6_ERROR_UDP_CHECKSUM);
1146           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1147                   IP6_ERROR_ICMP_CHECKSUM);
1148           error0 = (!good_l4_csum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0);
1149           error1 = (!good_l4_csum1 ? IP6_ERROR_UDP_CHECKSUM + type1 : error1);
1150
1151           /* Drop packets from unroutable hosts. */
1152           /* If this is a neighbor solicitation (ICMP), skip source RPF check */
1153           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1154               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1155               !ip6_address_is_link_local_unicast (&ip0->src_address))
1156             {
1157               error0 = (!ip6_urpf_loose_check (im, p0, ip0)
1158                         ? IP6_ERROR_SRC_LOOKUP_MISS : error0);
1159             }
1160           if (error1 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1161               type1 != IP_BUILTIN_PROTOCOL_ICMP &&
1162               !ip6_address_is_link_local_unicast (&ip1->src_address))
1163             {
1164               error1 = (!ip6_urpf_loose_check (im, p1, ip1)
1165                         ? IP6_ERROR_SRC_LOOKUP_MISS : error1);
1166             }
1167
1168           /* TODO maybe move to lookup? */
1169           vnet_buffer (p0)->ip.fib_index =
1170             vec_elt (im->fib_index_by_sw_if_index,
1171                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
1172           vnet_buffer (p0)->ip.fib_index =
1173             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1174              (u32) ~ 0) ? vnet_buffer (p0)->ip.
1175             fib_index : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1176
1177           vnet_buffer (p1)->ip.fib_index =
1178             vec_elt (im->fib_index_by_sw_if_index,
1179                      vnet_buffer (p1)->sw_if_index[VLIB_RX]);
1180           vnet_buffer (p1)->ip.fib_index =
1181             (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
1182              (u32) ~ 0) ? vnet_buffer (p1)->ip.
1183             fib_index : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1184
1185
1186         skip_checks:
1187
1188           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1189           next1 = lm->local_next_by_ip_protocol[ip1->protocol];
1190
1191           next0 =
1192             error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1193           next1 =
1194             error1 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1195
1196           p0->error = error_node->errors[error0];
1197           p1->error = error_node->errors[error1];
1198
1199           if (head_of_feature_arc)
1200             {
1201               if (PREDICT_TRUE (error0 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1202                 vnet_feature_arc_start (arc_index,
1203                                         vnet_buffer (p0)->sw_if_index
1204                                         [VLIB_RX], &next0, p0);
1205               if (PREDICT_TRUE (error1 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1206                 vnet_feature_arc_start (arc_index,
1207                                         vnet_buffer (p1)->sw_if_index
1208                                         [VLIB_RX], &next1, p1);
1209             }
1210
1211           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1212                                            to_next, n_left_to_next,
1213                                            pi0, pi1, next0, next1);
1214         }
1215
1216       while (n_left_from > 0 && n_left_to_next > 0)
1217         {
1218           vlib_buffer_t *p0;
1219           ip6_header_t *ip0;
1220           udp_header_t *udp0;
1221           u32 pi0, ip_len0, udp_len0, flags0, next0;
1222           i32 len_diff0;
1223           u8 error0, type0, good_l4_csum0;
1224           u32 udp_offset0;
1225           u8 is_tcp_udp0;
1226
1227           pi0 = to_next[0] = from[0];
1228           from += 1;
1229           n_left_from -= 1;
1230           to_next += 1;
1231           n_left_to_next -= 1;
1232
1233           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1234
1235           p0 = vlib_get_buffer (vm, pi0);
1236           ip0 = vlib_buffer_get_current (p0);
1237
1238           if (head_of_feature_arc == 0)
1239             goto skip_check;
1240
1241           vnet_buffer (p0)->l3_hdr_offset = p0->current_data;
1242
1243           type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol];
1244           flags0 = p0->flags;
1245           is_tcp_udp0 = ip6_next_proto_is_tcp_udp (p0, ip0, &udp_offset0);
1246           good_l4_csum0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT
1247                            || (flags0 & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
1248                                || flags0 & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
1249             != 0;
1250
1251           len_diff0 = 0;
1252           if (PREDICT_TRUE (is_tcp_udp0))
1253             {
1254               udp0 = (udp_header_t *) ((u8 *) ip0 + udp_offset0);
1255               /* Don't verify UDP checksum for packets with explicit zero
1256                * checksum. */
1257               good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UDP
1258                 && udp0->checksum == 0;
1259               /* Verify UDP length. */
1260               if (is_tcp_udp0 == IP_PROTOCOL_UDP)
1261                 {
1262                   ip_len0 = clib_net_to_host_u16 (ip0->payload_length);
1263                   udp_len0 = clib_net_to_host_u16 (udp0->length);
1264                   len_diff0 = ip_len0 - udp_len0;
1265                 }
1266             }
1267
1268           good_l4_csum0 |= type0 == IP_BUILTIN_PROTOCOL_UNKNOWN;
1269           len_diff0 = type0 == IP_BUILTIN_PROTOCOL_UDP ? len_diff0 : 0;
1270
1271           if (PREDICT_FALSE (type0 != IP_BUILTIN_PROTOCOL_UNKNOWN
1272                              && !good_l4_csum0
1273                              && !(flags0 &
1274                                   VNET_BUFFER_F_L4_CHECKSUM_COMPUTED)))
1275             {
1276               flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, p0);
1277               good_l4_csum0 =
1278                 (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1279             }
1280
1281           error0 = IP6_ERROR_UNKNOWN_PROTOCOL;
1282           error0 = len_diff0 < 0 ? IP6_ERROR_UDP_LENGTH : error0;
1283
1284           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
1285                   IP6_ERROR_UDP_CHECKSUM);
1286           ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_ICMP ==
1287                   IP6_ERROR_ICMP_CHECKSUM);
1288           error0 = (!good_l4_csum0 ? IP6_ERROR_UDP_CHECKSUM + type0 : error0);
1289
1290           /* If this is a neighbor solicitation (ICMP), skip src RPF check */
1291           if (error0 == IP6_ERROR_UNKNOWN_PROTOCOL &&
1292               type0 != IP_BUILTIN_PROTOCOL_ICMP &&
1293               !ip6_address_is_link_local_unicast (&ip0->src_address))
1294             {
1295               error0 = (!ip6_urpf_loose_check (im, p0, ip0)
1296                         ? IP6_ERROR_SRC_LOOKUP_MISS : error0);
1297             }
1298
1299           vnet_buffer (p0)->ip.fib_index =
1300             vec_elt (im->fib_index_by_sw_if_index,
1301                      vnet_buffer (p0)->sw_if_index[VLIB_RX]);
1302           vnet_buffer (p0)->ip.fib_index =
1303             (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
1304              (u32) ~ 0) ? vnet_buffer (p0)->ip.
1305             fib_index : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1306
1307         skip_check:
1308
1309           next0 = lm->local_next_by_ip_protocol[ip0->protocol];
1310           next0 =
1311             error0 != IP6_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1312
1313           p0->error = error_node->errors[error0];
1314
1315           if (head_of_feature_arc)
1316             {
1317               if (PREDICT_TRUE (error0 == (u8) IP6_ERROR_UNKNOWN_PROTOCOL))
1318                 vnet_feature_arc_start (arc_index,
1319                                         vnet_buffer (p0)->sw_if_index
1320                                         [VLIB_RX], &next0, p0);
1321             }
1322
1323           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1324                                            to_next, n_left_to_next,
1325                                            pi0, next0);
1326         }
1327
1328       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1329     }
1330
1331   return frame->n_vectors;
1332 }
1333
1334 static uword
1335 ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1336 {
1337   return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
1338 }
1339
1340 /* *INDENT-OFF* */
1341 VLIB_REGISTER_NODE (ip6_local_node, static) =
1342 {
1343   .function = ip6_local,
1344   .name = "ip6-local",
1345   .vector_size = sizeof (u32),
1346   .format_trace = format_ip6_forward_next_trace,
1347   .n_next_nodes = IP_LOCAL_N_NEXT,
1348   .next_nodes =
1349   {
1350     [IP_LOCAL_NEXT_DROP] = "ip6-drop",
1351     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
1352     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
1353     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
1354     [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-reassembly",
1355   },
1356 };
1357 /* *INDENT-ON* */
1358
1359 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local);
1360
1361
1362 static uword
1363 ip6_local_end_of_arc (vlib_main_t * vm,
1364                       vlib_node_runtime_t * node, vlib_frame_t * frame)
1365 {
1366   return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
1367 }
1368
1369 /* *INDENT-OFF* */
1370 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node,static) = {
1371   .function = ip6_local_end_of_arc,
1372   .name = "ip6-local-end-of-arc",
1373   .vector_size = sizeof (u32),
1374
1375   .format_trace = format_ip6_forward_next_trace,
1376   .sibling_of = "ip6-local",
1377 };
1378
1379 VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_end_of_arc_node, ip6_local_end_of_arc)
1380
1381 VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
1382   .arc_name = "ip6-local",
1383   .node_name = "ip6-local-end-of-arc",
1384   .runs_before = 0, /* not before any other features */
1385 };
1386 /* *INDENT-ON* */
1387
1388 void
1389 ip6_register_protocol (u32 protocol, u32 node_index)
1390 {
1391   vlib_main_t *vm = vlib_get_main ();
1392   ip6_main_t *im = &ip6_main;
1393   ip_lookup_main_t *lm = &im->lookup_main;
1394
1395   ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1396   lm->local_next_by_ip_protocol[protocol] =
1397     vlib_node_add_next (vm, ip6_local_node.index, node_index);
1398 }
1399
1400 typedef enum
1401 {
1402   IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1403   IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX,
1404   IP6_DISCOVER_NEIGHBOR_N_NEXT,
1405 } ip6_discover_neighbor_next_t;
1406
1407 typedef enum
1408 {
1409   IP6_DISCOVER_NEIGHBOR_ERROR_DROP,
1410   IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT,
1411   IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS,
1412 } ip6_discover_neighbor_error_t;
1413
1414 static uword
1415 ip6_discover_neighbor_inline (vlib_main_t * vm,
1416                               vlib_node_runtime_t * node,
1417                               vlib_frame_t * frame, int is_glean)
1418 {
1419   vnet_main_t *vnm = vnet_get_main ();
1420   ip6_main_t *im = &ip6_main;
1421   ip_lookup_main_t *lm = &im->lookup_main;
1422   u32 *from, *to_next_drop;
1423   uword n_left_from, n_left_to_next_drop;
1424   static f64 time_last_seed_change = -1e100;
1425   static u32 hash_seeds[3];
1426   static uword hash_bitmap[256 / BITS (uword)];
1427   f64 time_now;
1428   int bogus_length;
1429
1430   if (node->flags & VLIB_NODE_FLAG_TRACE)
1431     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
1432
1433   time_now = vlib_time_now (vm);
1434   if (time_now - time_last_seed_change > 1e-3)
1435     {
1436       uword i;
1437       u32 *r = clib_random_buffer_get_data (&vm->random_buffer,
1438                                             sizeof (hash_seeds));
1439       for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1440         hash_seeds[i] = r[i];
1441
1442       /* Mark all hash keys as been not-seen before. */
1443       for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1444         hash_bitmap[i] = 0;
1445
1446       time_last_seed_change = time_now;
1447     }
1448
1449   from = vlib_frame_vector_args (frame);
1450   n_left_from = frame->n_vectors;
1451
1452   while (n_left_from > 0)
1453     {
1454       vlib_get_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1455                            to_next_drop, n_left_to_next_drop);
1456
1457       while (n_left_from > 0 && n_left_to_next_drop > 0)
1458         {
1459           vlib_buffer_t *p0;
1460           ip6_header_t *ip0;
1461           u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1462           uword bm0;
1463           ip_adjacency_t *adj0;
1464           vnet_hw_interface_t *hw_if0;
1465           u32 next0;
1466
1467           pi0 = from[0];
1468
1469           p0 = vlib_get_buffer (vm, pi0);
1470
1471           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1472
1473           ip0 = vlib_buffer_get_current (p0);
1474
1475           adj0 = adj_get (adj_index0);
1476
1477           if (!is_glean)
1478             {
1479               ip0->dst_address.as_u64[0] =
1480                 adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
1481               ip0->dst_address.as_u64[1] =
1482                 adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
1483             }
1484
1485           a0 = hash_seeds[0];
1486           b0 = hash_seeds[1];
1487           c0 = hash_seeds[2];
1488
1489           sw_if_index0 = adj0->rewrite_header.sw_if_index;
1490           vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1491
1492           a0 ^= sw_if_index0;
1493           b0 ^= ip0->dst_address.as_u32[0];
1494           c0 ^= ip0->dst_address.as_u32[1];
1495
1496           hash_v3_mix32 (a0, b0, c0);
1497
1498           b0 ^= ip0->dst_address.as_u32[2];
1499           c0 ^= ip0->dst_address.as_u32[3];
1500
1501           hash_v3_finalize32 (a0, b0, c0);
1502
1503           c0 &= BITS (hash_bitmap) - 1;
1504           c0 = c0 / BITS (uword);
1505           m0 = (uword) 1 << (c0 % BITS (uword));
1506
1507           bm0 = hash_bitmap[c0];
1508           drop0 = (bm0 & m0) != 0;
1509
1510           /* Mark it as seen. */
1511           hash_bitmap[c0] = bm0 | m0;
1512
1513           from += 1;
1514           n_left_from -= 1;
1515           to_next_drop[0] = pi0;
1516           to_next_drop += 1;
1517           n_left_to_next_drop -= 1;
1518
1519           hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1520
1521           /* If the interface is link-down, drop the pkt */
1522           if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
1523             drop0 = 1;
1524
1525           p0->error =
1526             node->errors[drop0 ? IP6_DISCOVER_NEIGHBOR_ERROR_DROP
1527                          : IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT];
1528           if (drop0)
1529             continue;
1530
1531           /*
1532            * the adj has been updated to a rewrite but the node the DPO that got
1533            * us here hasn't - yet. no big deal. we'll drop while we wait.
1534            */
1535           if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
1536             continue;
1537
1538           {
1539             u32 bi0 = 0;
1540             icmp6_neighbor_solicitation_header_t *h0;
1541             vlib_buffer_t *b0;
1542
1543             h0 = vlib_packet_template_get_packet
1544               (vm, &im->discover_neighbor_packet_template, &bi0);
1545
1546             /*
1547              * Build ethernet header.
1548              * Choose source address based on destination lookup
1549              * adjacency.
1550              */
1551             if (ip6_src_address_for_packet (lm,
1552                                             sw_if_index0,
1553                                             &h0->ip.src_address))
1554               {
1555                 /* There is no address on the interface */
1556                 p0->error =
1557                   node->errors[IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS];
1558                 vlib_buffer_free (vm, &bi0, 1);
1559                 continue;
1560               }
1561
1562             /*
1563              * Destination address is a solicited node multicast address.
1564              * We need to fill in
1565              * the low 24 bits with low 24 bits of target's address.
1566              */
1567             h0->ip.dst_address.as_u8[13] = ip0->dst_address.as_u8[13];
1568             h0->ip.dst_address.as_u8[14] = ip0->dst_address.as_u8[14];
1569             h0->ip.dst_address.as_u8[15] = ip0->dst_address.as_u8[15];
1570
1571             h0->neighbor.target_address = ip0->dst_address;
1572
1573             clib_memcpy (h0->link_layer_option.ethernet_address,
1574                          hw_if0->hw_address, vec_len (hw_if0->hw_address));
1575
1576             /* $$$$ appears we need this; why is the checksum non-zero? */
1577             h0->neighbor.icmp.checksum = 0;
1578             h0->neighbor.icmp.checksum =
1579               ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h0->ip,
1580                                                  &bogus_length);
1581
1582             ASSERT (bogus_length == 0);
1583
1584             vlib_buffer_copy_trace_flag (vm, p0, bi0);
1585             b0 = vlib_get_buffer (vm, bi0);
1586             vnet_buffer (b0)->sw_if_index[VLIB_TX]
1587               = vnet_buffer (p0)->sw_if_index[VLIB_TX];
1588
1589             /* Add rewrite/encap string. */
1590             vnet_rewrite_one_header (adj0[0], h0, sizeof (ethernet_header_t));
1591             vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
1592
1593             next0 = IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX;
1594
1595             vlib_set_next_frame_buffer (vm, node, next0, bi0);
1596           }
1597         }
1598
1599       vlib_put_next_frame (vm, node, IP6_DISCOVER_NEIGHBOR_NEXT_DROP,
1600                            n_left_to_next_drop);
1601     }
1602
1603   return frame->n_vectors;
1604 }
1605
1606 static uword
1607 ip6_discover_neighbor (vlib_main_t * vm,
1608                        vlib_node_runtime_t * node, vlib_frame_t * frame)
1609 {
1610   return (ip6_discover_neighbor_inline (vm, node, frame, 0));
1611 }
1612
1613 static uword
1614 ip6_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
1615 {
1616   return (ip6_discover_neighbor_inline (vm, node, frame, 1));
1617 }
1618
1619 static char *ip6_discover_neighbor_error_strings[] = {
1620   [IP6_DISCOVER_NEIGHBOR_ERROR_DROP] = "address overflow drops",
1621   [IP6_DISCOVER_NEIGHBOR_ERROR_REQUEST_SENT] = "neighbor solicitations sent",
1622   [IP6_DISCOVER_NEIGHBOR_ERROR_NO_SOURCE_ADDRESS]
1623     = "no source address for ND solicitation",
1624 };
1625
1626 /* *INDENT-OFF* */
1627 VLIB_REGISTER_NODE (ip6_discover_neighbor_node) =
1628 {
1629   .function = ip6_discover_neighbor,
1630   .name = "ip6-discover-neighbor",
1631   .vector_size = sizeof (u32),
1632   .format_trace = format_ip6_forward_next_trace,
1633   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1634   .error_strings = ip6_discover_neighbor_error_strings,
1635   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1636   .next_nodes =
1637   {
1638     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "ip6-drop",
1639     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1640   },
1641 };
1642 /* *INDENT-ON* */
1643
1644 /* *INDENT-OFF* */
1645 VLIB_REGISTER_NODE (ip6_glean_node) =
1646 {
1647   .function = ip6_glean,
1648   .name = "ip6-glean",
1649   .vector_size = sizeof (u32),
1650   .format_trace = format_ip6_forward_next_trace,
1651   .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
1652   .error_strings = ip6_discover_neighbor_error_strings,
1653   .n_next_nodes = IP6_DISCOVER_NEIGHBOR_N_NEXT,
1654   .next_nodes =
1655   {
1656     [IP6_DISCOVER_NEIGHBOR_NEXT_DROP] = "ip6-drop",
1657     [IP6_DISCOVER_NEIGHBOR_NEXT_REPLY_TX] = "interface-output",
1658   },
1659 };
1660 /* *INDENT-ON* */
1661
1662 clib_error_t *
1663 ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
1664 {
1665   vnet_main_t *vnm = vnet_get_main ();
1666   ip6_main_t *im = &ip6_main;
1667   icmp6_neighbor_solicitation_header_t *h;
1668   ip6_address_t *src;
1669   ip_interface_address_t *ia;
1670   ip_adjacency_t *adj;
1671   vnet_hw_interface_t *hi;
1672   vnet_sw_interface_t *si;
1673   vlib_buffer_t *b;
1674   adj_index_t ai;
1675   u32 bi = 0;
1676   int bogus_length;
1677
1678   si = vnet_get_sw_interface (vnm, sw_if_index);
1679
1680   if (!(si->flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
1681     {
1682       return clib_error_return (0, "%U: interface %U down",
1683                                 format_ip6_address, dst,
1684                                 format_vnet_sw_if_index_name, vnm,
1685                                 sw_if_index);
1686     }
1687
1688   src =
1689     ip6_interface_address_matching_destination (im, dst, sw_if_index, &ia);
1690   if (!src)
1691     {
1692       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
1693       return clib_error_return
1694         (0, "no matching interface address for destination %U (interface %U)",
1695          format_ip6_address, dst,
1696          format_vnet_sw_if_index_name, vnm, sw_if_index);
1697     }
1698
1699   h =
1700     vlib_packet_template_get_packet (vm,
1701                                      &im->discover_neighbor_packet_template,
1702                                      &bi);
1703
1704   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1705
1706   /* Destination address is a solicited node multicast address.  We need to fill in
1707      the low 24 bits with low 24 bits of target's address. */
1708   h->ip.dst_address.as_u8[13] = dst->as_u8[13];
1709   h->ip.dst_address.as_u8[14] = dst->as_u8[14];
1710   h->ip.dst_address.as_u8[15] = dst->as_u8[15];
1711
1712   h->ip.src_address = src[0];
1713   h->neighbor.target_address = dst[0];
1714
1715   if (PREDICT_FALSE (!hi->hw_address))
1716     {
1717       return clib_error_return (0, "%U: interface %U do not support ip probe",
1718                                 format_ip6_address, dst,
1719                                 format_vnet_sw_if_index_name, vnm,
1720                                 sw_if_index);
1721     }
1722
1723   clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address,
1724                vec_len (hi->hw_address));
1725
1726   h->neighbor.icmp.checksum =
1727     ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
1728   ASSERT (bogus_length == 0);
1729
1730   b = vlib_get_buffer (vm, bi);
1731   vnet_buffer (b)->sw_if_index[VLIB_RX] =
1732     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
1733
1734   /* Add encapsulation string for software interface (e.g. ethernet header). */
1735   ip46_address_t nh = {
1736     .ip6 = *dst,
1737   };
1738
1739   ai = adj_nbr_add_or_lock (FIB_PROTOCOL_IP6,
1740                             VNET_LINK_IP6, &nh, sw_if_index);
1741   adj = adj_get (ai);
1742
1743   /* Peer has been previously resolved, retrieve glean adj instead */
1744   if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
1745     {
1746       adj_unlock (ai);
1747       ai = adj_glean_add_or_lock (FIB_PROTOCOL_IP6,
1748                                   VNET_LINK_IP6, sw_if_index, &nh);
1749       adj = adj_get (ai);
1750     }
1751
1752   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
1753   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
1754
1755   {
1756     vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
1757     u32 *to_next = vlib_frame_vector_args (f);
1758     to_next[0] = bi;
1759     f->n_vectors = 1;
1760     vlib_put_frame_to_node (vm, hi->output_node_index, f);
1761   }
1762
1763   adj_unlock (ai);
1764   return /* no error */ 0;
1765 }
1766
1767 typedef enum
1768 {
1769   IP6_REWRITE_NEXT_DROP,
1770   IP6_REWRITE_NEXT_ICMP_ERROR,
1771 } ip6_rewrite_next_t;
1772
1773 always_inline uword
1774 ip6_rewrite_inline (vlib_main_t * vm,
1775                     vlib_node_runtime_t * node,
1776                     vlib_frame_t * frame,
1777                     int do_counters, int is_midchain, int is_mcast)
1778 {
1779   ip_lookup_main_t *lm = &ip6_main.lookup_main;
1780   u32 *from = vlib_frame_vector_args (frame);
1781   u32 n_left_from, n_left_to_next, *to_next, next_index;
1782   vlib_node_runtime_t *error_node =
1783     vlib_node_get_runtime (vm, ip6_input_node.index);
1784
1785   n_left_from = frame->n_vectors;
1786   next_index = node->cached_next_index;
1787   u32 thread_index = vlib_get_thread_index ();
1788
1789   while (n_left_from > 0)
1790     {
1791       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1792
1793       while (n_left_from >= 4 && n_left_to_next >= 2)
1794         {
1795           ip_adjacency_t *adj0, *adj1;
1796           vlib_buffer_t *p0, *p1;
1797           ip6_header_t *ip0, *ip1;
1798           u32 pi0, rw_len0, next0, error0, adj_index0;
1799           u32 pi1, rw_len1, next1, error1, adj_index1;
1800           u32 tx_sw_if_index0, tx_sw_if_index1;
1801
1802           /* Prefetch next iteration. */
1803           {
1804             vlib_buffer_t *p2, *p3;
1805
1806             p2 = vlib_get_buffer (vm, from[2]);
1807             p3 = vlib_get_buffer (vm, from[3]);
1808
1809             vlib_prefetch_buffer_header (p2, LOAD);
1810             vlib_prefetch_buffer_header (p3, LOAD);
1811
1812             CLIB_PREFETCH (p2->pre_data, 32, STORE);
1813             CLIB_PREFETCH (p3->pre_data, 32, STORE);
1814
1815             CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
1816             CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
1817           }
1818
1819           pi0 = to_next[0] = from[0];
1820           pi1 = to_next[1] = from[1];
1821
1822           from += 2;
1823           n_left_from -= 2;
1824           to_next += 2;
1825           n_left_to_next -= 2;
1826
1827           p0 = vlib_get_buffer (vm, pi0);
1828           p1 = vlib_get_buffer (vm, pi1);
1829
1830           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1831           adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
1832
1833           ip0 = vlib_buffer_get_current (p0);
1834           ip1 = vlib_buffer_get_current (p1);
1835
1836           error0 = error1 = IP6_ERROR_NONE;
1837           next0 = next1 = IP6_REWRITE_NEXT_DROP;
1838
1839           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
1840             {
1841               i32 hop_limit0 = ip0->hop_limit;
1842
1843               /* Input node should have reject packets with hop limit 0. */
1844               ASSERT (ip0->hop_limit > 0);
1845
1846               hop_limit0 -= 1;
1847
1848               ip0->hop_limit = hop_limit0;
1849
1850               /*
1851                * If the hop count drops below 1 when forwarding, generate
1852                * an ICMP response.
1853                */
1854               if (PREDICT_FALSE (hop_limit0 <= 0))
1855                 {
1856                   error0 = IP6_ERROR_TIME_EXPIRED;
1857                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
1858                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1859                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
1860                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1861                                                0);
1862                 }
1863             }
1864           else
1865             {
1866               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1867             }
1868           if (PREDICT_TRUE (!(p1->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
1869             {
1870               i32 hop_limit1 = ip1->hop_limit;
1871
1872               /* Input node should have reject packets with hop limit 0. */
1873               ASSERT (ip1->hop_limit > 0);
1874
1875               hop_limit1 -= 1;
1876
1877               ip1->hop_limit = hop_limit1;
1878
1879               /*
1880                * If the hop count drops below 1 when forwarding, generate
1881                * an ICMP response.
1882                */
1883               if (PREDICT_FALSE (hop_limit1 <= 0))
1884                 {
1885                   error1 = IP6_ERROR_TIME_EXPIRED;
1886                   next1 = IP6_REWRITE_NEXT_ICMP_ERROR;
1887                   vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1888                   icmp6_error_set_vnet_buffer (p1, ICMP6_time_exceeded,
1889                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
1890                                                0);
1891                 }
1892             }
1893           else
1894             {
1895               p1->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
1896             }
1897           adj0 = adj_get (adj_index0);
1898           adj1 = adj_get (adj_index1);
1899
1900           rw_len0 = adj0[0].rewrite_header.data_bytes;
1901           rw_len1 = adj1[0].rewrite_header.data_bytes;
1902           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
1903           vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
1904
1905           if (do_counters)
1906             {
1907               vlib_increment_combined_counter
1908                 (&adjacency_counters,
1909                  thread_index, adj_index0, 1,
1910                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
1911               vlib_increment_combined_counter
1912                 (&adjacency_counters,
1913                  thread_index, adj_index1, 1,
1914                  vlib_buffer_length_in_chain (vm, p1) + rw_len1);
1915             }
1916
1917           /* Check MTU of outgoing interface. */
1918           error0 =
1919             (vlib_buffer_length_in_chain (vm, p0) >
1920              adj0[0].
1921              rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED :
1922              error0);
1923           error1 =
1924             (vlib_buffer_length_in_chain (vm, p1) >
1925              adj1[0].
1926              rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED :
1927              error1);
1928
1929           /* Don't adjust the buffer for hop count issue; icmp-error node
1930            * wants to see the IP headerr */
1931           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
1932             {
1933               p0->current_data -= rw_len0;
1934               p0->current_length += rw_len0;
1935
1936               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
1937               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
1938               next0 = adj0[0].rewrite_header.next_index;
1939
1940               if (PREDICT_FALSE
1941                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1942                 vnet_feature_arc_start (lm->output_feature_arc_index,
1943                                         tx_sw_if_index0, &next0, p0);
1944             }
1945           if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
1946             {
1947               p1->current_data -= rw_len1;
1948               p1->current_length += rw_len1;
1949
1950               tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
1951               vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
1952               next1 = adj1[0].rewrite_header.next_index;
1953
1954               if (PREDICT_FALSE
1955                   (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
1956                 vnet_feature_arc_start (lm->output_feature_arc_index,
1957                                         tx_sw_if_index1, &next1, p1);
1958             }
1959
1960           /* Guess we are only writing on simple Ethernet header. */
1961           vnet_rewrite_two_headers (adj0[0], adj1[0],
1962                                     ip0, ip1, sizeof (ethernet_header_t));
1963
1964           if (is_midchain)
1965             {
1966               adj0->sub_type.midchain.fixup_func
1967                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
1968               adj1->sub_type.midchain.fixup_func
1969                 (vm, adj1, p1, adj1->sub_type.midchain.fixup_data);
1970             }
1971           if (is_mcast)
1972             {
1973               /*
1974                * copy bytes from the IP address into the MAC rewrite
1975                */
1976               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
1977               vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
1978             }
1979
1980           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1981                                            to_next, n_left_to_next,
1982                                            pi0, pi1, next0, next1);
1983         }
1984
1985       while (n_left_from > 0 && n_left_to_next > 0)
1986         {
1987           ip_adjacency_t *adj0;
1988           vlib_buffer_t *p0;
1989           ip6_header_t *ip0;
1990           u32 pi0, rw_len0;
1991           u32 adj_index0, next0, error0;
1992           u32 tx_sw_if_index0;
1993
1994           pi0 = to_next[0] = from[0];
1995
1996           p0 = vlib_get_buffer (vm, pi0);
1997
1998           adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1999
2000           adj0 = adj_get (adj_index0);
2001
2002           ip0 = vlib_buffer_get_current (p0);
2003
2004           error0 = IP6_ERROR_NONE;
2005           next0 = IP6_REWRITE_NEXT_DROP;
2006
2007           /* Check hop limit */
2008           if (PREDICT_TRUE (!(p0->flags & VNET_BUFFER_F_LOCALLY_ORIGINATED)))
2009             {
2010               i32 hop_limit0 = ip0->hop_limit;
2011
2012               ASSERT (ip0->hop_limit > 0);
2013
2014               hop_limit0 -= 1;
2015
2016               ip0->hop_limit = hop_limit0;
2017
2018               if (PREDICT_FALSE (hop_limit0 <= 0))
2019                 {
2020                   /*
2021                    * If the hop count drops below 1 when forwarding, generate
2022                    * an ICMP response.
2023                    */
2024                   error0 = IP6_ERROR_TIME_EXPIRED;
2025                   next0 = IP6_REWRITE_NEXT_ICMP_ERROR;
2026                   vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2027                   icmp6_error_set_vnet_buffer (p0, ICMP6_time_exceeded,
2028                                                ICMP6_time_exceeded_ttl_exceeded_in_transit,
2029                                                0);
2030                 }
2031             }
2032           else
2033             {
2034               p0->flags &= ~VNET_BUFFER_F_LOCALLY_ORIGINATED;
2035             }
2036
2037           /* Guess we are only writing on simple Ethernet header. */
2038           vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2039
2040           /* Update packet buffer attributes/set output interface. */
2041           rw_len0 = adj0[0].rewrite_header.data_bytes;
2042           vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2043
2044           if (do_counters)
2045             {
2046               vlib_increment_combined_counter
2047                 (&adjacency_counters,
2048                  thread_index, adj_index0, 1,
2049                  vlib_buffer_length_in_chain (vm, p0) + rw_len0);
2050             }
2051
2052           /* Check MTU of outgoing interface. */
2053           error0 =
2054             (vlib_buffer_length_in_chain (vm, p0) >
2055              adj0[0].
2056              rewrite_header.max_l3_packet_bytes ? IP6_ERROR_MTU_EXCEEDED :
2057              error0);
2058
2059           /* Don't adjust the buffer for hop count issue; icmp-error node
2060            * wants to see the IP headerr */
2061           if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
2062             {
2063               p0->current_data -= rw_len0;
2064               p0->current_length += rw_len0;
2065
2066               tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2067
2068               vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2069               next0 = adj0[0].rewrite_header.next_index;
2070
2071               if (PREDICT_FALSE
2072                   (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
2073                 vnet_feature_arc_start (lm->output_feature_arc_index,
2074                                         tx_sw_if_index0, &next0, p0);
2075             }
2076
2077           if (is_midchain)
2078             {
2079               adj0->sub_type.midchain.fixup_func
2080                 (vm, adj0, p0, adj0->sub_type.midchain.fixup_data);
2081             }
2082           if (is_mcast)
2083             {
2084               vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
2085             }
2086
2087           p0->error = error_node->errors[error0];
2088
2089           from += 1;
2090           n_left_from -= 1;
2091           to_next += 1;
2092           n_left_to_next -= 1;
2093
2094           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2095                                            to_next, n_left_to_next,
2096                                            pi0, next0);
2097         }
2098
2099       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2100     }
2101
2102   /* Need to do trace after rewrites to pick up new packet data. */
2103   if (node->flags & VLIB_NODE_FLAG_TRACE)
2104     ip6_forward_next_trace (vm, node, frame, VLIB_TX);
2105
2106   return frame->n_vectors;
2107 }
2108
2109 static uword
2110 ip6_rewrite (vlib_main_t * vm,
2111              vlib_node_runtime_t * node, vlib_frame_t * frame)
2112 {
2113   if (adj_are_counters_enabled ())
2114     return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
2115   else
2116     return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
2117 }
2118
2119 static uword
2120 ip6_rewrite_mcast (vlib_main_t * vm,
2121                    vlib_node_runtime_t * node, vlib_frame_t * frame)
2122 {
2123   if (adj_are_counters_enabled ())
2124     return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
2125   else
2126     return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
2127 }
2128
2129 static uword
2130 ip6_midchain (vlib_main_t * vm,
2131               vlib_node_runtime_t * node, vlib_frame_t * frame)
2132 {
2133   if (adj_are_counters_enabled ())
2134     return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
2135   else
2136     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
2137 }
2138
2139 static uword
2140 ip6_mcast_midchain (vlib_main_t * vm,
2141                     vlib_node_runtime_t * node, vlib_frame_t * frame)
2142 {
2143   if (adj_are_counters_enabled ())
2144     return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
2145   else
2146     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
2147 }
2148
2149 /* *INDENT-OFF* */
2150 VLIB_REGISTER_NODE (ip6_midchain_node) =
2151 {
2152   .function = ip6_midchain,
2153   .name = "ip6-midchain",
2154   .vector_size = sizeof (u32),
2155   .format_trace = format_ip6_forward_next_trace,
2156   .sibling_of = "ip6-rewrite",
2157   };
2158 /* *INDENT-ON* */
2159
2160 VLIB_NODE_FUNCTION_MULTIARCH (ip6_midchain_node, ip6_midchain);
2161
2162 /* *INDENT-OFF* */
2163 VLIB_REGISTER_NODE (ip6_rewrite_node) =
2164 {
2165   .function = ip6_rewrite,
2166   .name = "ip6-rewrite",
2167   .vector_size = sizeof (u32),
2168   .format_trace = format_ip6_rewrite_trace,
2169   .n_next_nodes = 2,
2170   .next_nodes =
2171   {
2172     [IP6_REWRITE_NEXT_DROP] = "ip6-drop",
2173     [IP6_REWRITE_NEXT_ICMP_ERROR] = "ip6-icmp-error",
2174   },
2175 };
2176 /* *INDENT-ON* */
2177
2178 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite);
2179
2180 /* *INDENT-OFF* */
2181 VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
2182 {
2183   .function = ip6_rewrite_mcast,
2184   .name = "ip6-rewrite-mcast",
2185   .vector_size = sizeof (u32),
2186   .format_trace = format_ip6_rewrite_trace,
2187   .sibling_of = "ip6-rewrite",
2188 };
2189 /* *INDENT-ON* */
2190
2191 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast);
2192
2193 /* *INDENT-OFF* */
2194 VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) =
2195 {
2196   .function = ip6_mcast_midchain,
2197   .name = "ip6-mcast-midchain",
2198   .vector_size = sizeof (u32),
2199   .format_trace = format_ip6_rewrite_trace,
2200   .sibling_of = "ip6-rewrite",
2201 };
2202 /* *INDENT-ON* */
2203
2204 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain);
2205
2206 /*
2207  * Hop-by-Hop handling
2208  */
2209 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
2210
2211 #define foreach_ip6_hop_by_hop_error \
2212 _(PROCESSED, "pkts with ip6 hop-by-hop options") \
2213 _(FORMAT, "incorrectly formatted hop-by-hop options") \
2214 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
2215
2216 /* *INDENT-OFF* */
2217 typedef enum
2218 {
2219 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
2220   foreach_ip6_hop_by_hop_error
2221 #undef _
2222   IP6_HOP_BY_HOP_N_ERROR,
2223 } ip6_hop_by_hop_error_t;
2224 /* *INDENT-ON* */
2225
2226 /*
2227  * Primary h-b-h handler trace support
2228  * We work pretty hard on the problem for obvious reasons
2229  */
2230 typedef struct
2231 {
2232   u32 next_index;
2233   u32 trace_len;
2234   u8 option_data[256];
2235 } ip6_hop_by_hop_trace_t;
2236
2237 vlib_node_registration_t ip6_hop_by_hop_node;
2238
2239 static char *ip6_hop_by_hop_error_strings[] = {
2240 #define _(sym,string) string,
2241   foreach_ip6_hop_by_hop_error
2242 #undef _
2243 };
2244
2245 u8 *
2246 format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
2247 {
2248   ip6_hop_by_hop_header_t *hbh0 = va_arg (*args, ip6_hop_by_hop_header_t *);
2249   int total_len = va_arg (*args, int);
2250   ip6_hop_by_hop_option_t *opt0, *limit0;
2251   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2252   u8 type0;
2253
2254   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
2255               hbh0->protocol, (hbh0->length + 1) << 3, total_len);
2256
2257   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2258   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 + total_len);
2259
2260   while (opt0 < limit0)
2261     {
2262       type0 = opt0->type;
2263       switch (type0)
2264         {
2265         case 0:         /* Pad, just stop */
2266           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0 + 1);
2267           break;
2268
2269         default:
2270           if (hm->trace[type0])
2271             {
2272               s = (*hm->trace[type0]) (s, opt0);
2273             }
2274           else
2275             {
2276               s =
2277                 format (s, "\n    unrecognized option %d length %d", type0,
2278                         opt0->length);
2279             }
2280           opt0 =
2281             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2282                                          sizeof (ip6_hop_by_hop_option_t));
2283           break;
2284         }
2285     }
2286   return s;
2287 }
2288
2289 static u8 *
2290 format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
2291 {
2292   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
2293   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
2294   ip6_hop_by_hop_trace_t *t = va_arg (*args, ip6_hop_by_hop_trace_t *);
2295   ip6_hop_by_hop_header_t *hbh0;
2296   ip6_hop_by_hop_option_t *opt0, *limit0;
2297   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2298
2299   u8 type0;
2300
2301   hbh0 = (ip6_hop_by_hop_header_t *) t->option_data;
2302
2303   s = format (s, "IP6_HOP_BY_HOP: next index %d len %d traced %d",
2304               t->next_index, (hbh0->length + 1) << 3, t->trace_len);
2305
2306   opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2307   limit0 = (ip6_hop_by_hop_option_t *) ((u8 *) hbh0) + t->trace_len;
2308
2309   while (opt0 < limit0)
2310     {
2311       type0 = opt0->type;
2312       switch (type0)
2313         {
2314         case 0:         /* Pad, just stop */
2315           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2316           break;
2317
2318         default:
2319           if (hm->trace[type0])
2320             {
2321               s = (*hm->trace[type0]) (s, opt0);
2322             }
2323           else
2324             {
2325               s =
2326                 format (s, "\n    unrecognized option %d length %d", type0,
2327                         opt0->length);
2328             }
2329           opt0 =
2330             (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2331                                          sizeof (ip6_hop_by_hop_option_t));
2332           break;
2333         }
2334     }
2335   return s;
2336 }
2337
2338 always_inline u8
2339 ip6_scan_hbh_options (vlib_buffer_t * b0,
2340                       ip6_header_t * ip0,
2341                       ip6_hop_by_hop_header_t * hbh0,
2342                       ip6_hop_by_hop_option_t * opt0,
2343                       ip6_hop_by_hop_option_t * limit0, u32 * next0)
2344 {
2345   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2346   u8 type0;
2347   u8 error0 = 0;
2348
2349   while (opt0 < limit0)
2350     {
2351       type0 = opt0->type;
2352       switch (type0)
2353         {
2354         case 0:         /* Pad1 */
2355           opt0 = (ip6_hop_by_hop_option_t *) ((u8 *) opt0) + 1;
2356           continue;
2357         case 1:         /* PadN */
2358           break;
2359         default:
2360           if (hm->options[type0])
2361             {
2362               if ((*hm->options[type0]) (b0, ip0, opt0) < 0)
2363                 {
2364                   error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2365                   return (error0);
2366                 }
2367             }
2368           else
2369             {
2370               /* Unrecognized mandatory option, check the two high order bits */
2371               switch (opt0->type & HBH_OPTION_TYPE_HIGH_ORDER_BITS)
2372                 {
2373                 case HBH_OPTION_TYPE_SKIP_UNKNOWN:
2374                   break;
2375                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN:
2376                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2377                   *next0 = IP_LOOKUP_NEXT_DROP;
2378                   break;
2379                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP:
2380                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2381                   *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2382                   icmp6_error_set_vnet_buffer (b0, ICMP6_parameter_problem,
2383                                                ICMP6_parameter_problem_unrecognized_option,
2384                                                (u8 *) opt0 - (u8 *) ip0);
2385                   break;
2386                 case HBH_OPTION_TYPE_DISCARD_UNKNOWN_ICMP_NOT_MCAST:
2387                   error0 = IP6_HOP_BY_HOP_ERROR_UNKNOWN_OPTION;
2388                   if (!ip6_address_is_multicast (&ip0->dst_address))
2389                     {
2390                       *next0 = IP_LOOKUP_NEXT_ICMP_ERROR;
2391                       icmp6_error_set_vnet_buffer (b0,
2392                                                    ICMP6_parameter_problem,
2393                                                    ICMP6_parameter_problem_unrecognized_option,
2394                                                    (u8 *) opt0 - (u8 *) ip0);
2395                     }
2396                   else
2397                     {
2398                       *next0 = IP_LOOKUP_NEXT_DROP;
2399                     }
2400                   break;
2401                 }
2402               return (error0);
2403             }
2404         }
2405       opt0 =
2406         (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
2407                                      sizeof (ip6_hop_by_hop_option_t));
2408     }
2409   return (error0);
2410 }
2411
2412 /*
2413  * Process the Hop-by-Hop Options header
2414  */
2415 static uword
2416 ip6_hop_by_hop (vlib_main_t * vm,
2417                 vlib_node_runtime_t * node, vlib_frame_t * frame)
2418 {
2419   vlib_node_runtime_t *error_node =
2420     vlib_node_get_runtime (vm, ip6_hop_by_hop_node.index);
2421   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2422   u32 n_left_from, *from, *to_next;
2423   ip_lookup_next_t next_index;
2424
2425   from = vlib_frame_vector_args (frame);
2426   n_left_from = frame->n_vectors;
2427   next_index = node->cached_next_index;
2428
2429   while (n_left_from > 0)
2430     {
2431       u32 n_left_to_next;
2432
2433       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2434
2435       while (n_left_from >= 4 && n_left_to_next >= 2)
2436         {
2437           u32 bi0, bi1;
2438           vlib_buffer_t *b0, *b1;
2439           u32 next0, next1;
2440           ip6_header_t *ip0, *ip1;
2441           ip6_hop_by_hop_header_t *hbh0, *hbh1;
2442           ip6_hop_by_hop_option_t *opt0, *limit0, *opt1, *limit1;
2443           u8 error0 = 0, error1 = 0;
2444
2445           /* Prefetch next iteration. */
2446           {
2447             vlib_buffer_t *p2, *p3;
2448
2449             p2 = vlib_get_buffer (vm, from[2]);
2450             p3 = vlib_get_buffer (vm, from[3]);
2451
2452             vlib_prefetch_buffer_header (p2, LOAD);
2453             vlib_prefetch_buffer_header (p3, LOAD);
2454
2455             CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2456             CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
2457           }
2458
2459           /* Speculatively enqueue b0, b1 to the current next frame */
2460           to_next[0] = bi0 = from[0];
2461           to_next[1] = bi1 = from[1];
2462           from += 2;
2463           to_next += 2;
2464           n_left_from -= 2;
2465           n_left_to_next -= 2;
2466
2467           b0 = vlib_get_buffer (vm, bi0);
2468           b1 = vlib_get_buffer (vm, bi1);
2469
2470           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2471           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2472           ip_adjacency_t *adj0 = adj_get (adj_index0);
2473           u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
2474           ip_adjacency_t *adj1 = adj_get (adj_index1);
2475
2476           /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
2477           next0 = adj0->lookup_next_index;
2478           next1 = adj1->lookup_next_index;
2479
2480           ip0 = vlib_buffer_get_current (b0);
2481           ip1 = vlib_buffer_get_current (b1);
2482           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2483           hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1);
2484           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2485           opt1 = (ip6_hop_by_hop_option_t *) (hbh1 + 1);
2486           limit0 =
2487             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2488                                          ((hbh0->length + 1) << 3));
2489           limit1 =
2490             (ip6_hop_by_hop_option_t *) ((u8 *) hbh1 +
2491                                          ((hbh1->length + 1) << 3));
2492
2493           /*
2494            * Basic validity checks
2495            */
2496           if ((hbh0->length + 1) << 3 >
2497               clib_net_to_host_u16 (ip0->payload_length))
2498             {
2499               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2500               next0 = IP_LOOKUP_NEXT_DROP;
2501               goto outdual;
2502             }
2503           /* Scan the set of h-b-h options, process ones that we understand */
2504           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2505
2506           if ((hbh1->length + 1) << 3 >
2507               clib_net_to_host_u16 (ip1->payload_length))
2508             {
2509               error1 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2510               next1 = IP_LOOKUP_NEXT_DROP;
2511               goto outdual;
2512             }
2513           /* Scan the set of h-b-h options, process ones that we understand */
2514           error1 = ip6_scan_hbh_options (b1, ip1, hbh1, opt1, limit1, &next1);
2515
2516         outdual:
2517           /* Has the classifier flagged this buffer for special treatment? */
2518           if (PREDICT_FALSE
2519               ((error0 == 0)
2520                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2521             next0 = hm->next_override;
2522
2523           /* Has the classifier flagged this buffer for special treatment? */
2524           if (PREDICT_FALSE
2525               ((error1 == 0)
2526                && (vnet_buffer (b1)->l2_classify.opaque_index & OI_DECAP)))
2527             next1 = hm->next_override;
2528
2529           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
2530             {
2531               if (b0->flags & VLIB_BUFFER_IS_TRACED)
2532                 {
2533                   ip6_hop_by_hop_trace_t *t =
2534                     vlib_add_trace (vm, node, b0, sizeof (*t));
2535                   u32 trace_len = (hbh0->length + 1) << 3;
2536                   t->next_index = next0;
2537                   /* Capture the h-b-h option verbatim */
2538                   trace_len =
2539                     trace_len <
2540                     ARRAY_LEN (t->option_data) ? trace_len :
2541                     ARRAY_LEN (t->option_data);
2542                   t->trace_len = trace_len;
2543                   clib_memcpy (t->option_data, hbh0, trace_len);
2544                 }
2545               if (b1->flags & VLIB_BUFFER_IS_TRACED)
2546                 {
2547                   ip6_hop_by_hop_trace_t *t =
2548                     vlib_add_trace (vm, node, b1, sizeof (*t));
2549                   u32 trace_len = (hbh1->length + 1) << 3;
2550                   t->next_index = next1;
2551                   /* Capture the h-b-h option verbatim */
2552                   trace_len =
2553                     trace_len <
2554                     ARRAY_LEN (t->option_data) ? trace_len :
2555                     ARRAY_LEN (t->option_data);
2556                   t->trace_len = trace_len;
2557                   clib_memcpy (t->option_data, hbh1, trace_len);
2558                 }
2559
2560             }
2561
2562           b0->error = error_node->errors[error0];
2563           b1->error = error_node->errors[error1];
2564
2565           /* verify speculative enqueue, maybe switch current next frame */
2566           vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
2567                                            n_left_to_next, bi0, bi1, next0,
2568                                            next1);
2569         }
2570
2571       while (n_left_from > 0 && n_left_to_next > 0)
2572         {
2573           u32 bi0;
2574           vlib_buffer_t *b0;
2575           u32 next0;
2576           ip6_header_t *ip0;
2577           ip6_hop_by_hop_header_t *hbh0;
2578           ip6_hop_by_hop_option_t *opt0, *limit0;
2579           u8 error0 = 0;
2580
2581           /* Speculatively enqueue b0 to the current next frame */
2582           bi0 = from[0];
2583           to_next[0] = bi0;
2584           from += 1;
2585           to_next += 1;
2586           n_left_from -= 1;
2587           n_left_to_next -= 1;
2588
2589           b0 = vlib_get_buffer (vm, bi0);
2590           /*
2591            * Default use the next_index from the adjacency.
2592            * A HBH option rarely redirects to a different node
2593            */
2594           u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
2595           ip_adjacency_t *adj0 = adj_get (adj_index0);
2596           next0 = adj0->lookup_next_index;
2597
2598           ip0 = vlib_buffer_get_current (b0);
2599           hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1);
2600           opt0 = (ip6_hop_by_hop_option_t *) (hbh0 + 1);
2601           limit0 =
2602             (ip6_hop_by_hop_option_t *) ((u8 *) hbh0 +
2603                                          ((hbh0->length + 1) << 3));
2604
2605           /*
2606            * Basic validity checks
2607            */
2608           if ((hbh0->length + 1) << 3 >
2609               clib_net_to_host_u16 (ip0->payload_length))
2610             {
2611               error0 = IP6_HOP_BY_HOP_ERROR_FORMAT;
2612               next0 = IP_LOOKUP_NEXT_DROP;
2613               goto out0;
2614             }
2615
2616           /* Scan the set of h-b-h options, process ones that we understand */
2617           error0 = ip6_scan_hbh_options (b0, ip0, hbh0, opt0, limit0, &next0);
2618
2619         out0:
2620           /* Has the classifier flagged this buffer for special treatment? */
2621           if (PREDICT_FALSE
2622               ((error0 == 0)
2623                && (vnet_buffer (b0)->l2_classify.opaque_index & OI_DECAP)))
2624             next0 = hm->next_override;
2625
2626           if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
2627             {
2628               ip6_hop_by_hop_trace_t *t =
2629                 vlib_add_trace (vm, node, b0, sizeof (*t));
2630               u32 trace_len = (hbh0->length + 1) << 3;
2631               t->next_index = next0;
2632               /* Capture the h-b-h option verbatim */
2633               trace_len =
2634                 trace_len <
2635                 ARRAY_LEN (t->option_data) ? trace_len :
2636                 ARRAY_LEN (t->option_data);
2637               t->trace_len = trace_len;
2638               clib_memcpy (t->option_data, hbh0, trace_len);
2639             }
2640
2641           b0->error = error_node->errors[error0];
2642
2643           /* verify speculative enqueue, maybe switch current next frame */
2644           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
2645                                            n_left_to_next, bi0, next0);
2646         }
2647       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2648     }
2649   return frame->n_vectors;
2650 }
2651
2652 /* *INDENT-OFF* */
2653 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
2654 {
2655   .function = ip6_hop_by_hop,
2656   .name = "ip6-hop-by-hop",
2657   .sibling_of = "ip6-lookup",
2658   .vector_size = sizeof (u32),
2659   .format_trace = format_ip6_hop_by_hop_trace,
2660   .type = VLIB_NODE_TYPE_INTERNAL,
2661   .n_errors = ARRAY_LEN (ip6_hop_by_hop_error_strings),
2662   .error_strings = ip6_hop_by_hop_error_strings,
2663   .n_next_nodes = 0,
2664 };
2665 /* *INDENT-ON* */
2666
2667 VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop);
2668
2669 static clib_error_t *
2670 ip6_hop_by_hop_init (vlib_main_t * vm)
2671 {
2672   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2673   memset (hm->options, 0, sizeof (hm->options));
2674   memset (hm->trace, 0, sizeof (hm->trace));
2675   hm->next_override = IP6_LOOKUP_NEXT_POP_HOP_BY_HOP;
2676   return (0);
2677 }
2678
2679 VLIB_INIT_FUNCTION (ip6_hop_by_hop_init);
2680
2681 void
2682 ip6_hbh_set_next_override (uword next)
2683 {
2684   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2685
2686   hm->next_override = next;
2687 }
2688
2689 int
2690 ip6_hbh_register_option (u8 option,
2691                          int options (vlib_buffer_t * b, ip6_header_t * ip,
2692                                       ip6_hop_by_hop_option_t * opt),
2693                          u8 * trace (u8 * s, ip6_hop_by_hop_option_t * opt))
2694 {
2695   ip6_main_t *im = &ip6_main;
2696   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2697
2698   ASSERT (option < ARRAY_LEN (hm->options));
2699
2700   /* Already registered */
2701   if (hm->options[option])
2702     return (-1);
2703
2704   hm->options[option] = options;
2705   hm->trace[option] = trace;
2706
2707   /* Set global variable */
2708   im->hbh_enabled = 1;
2709
2710   return (0);
2711 }
2712
2713 int
2714 ip6_hbh_unregister_option (u8 option)
2715 {
2716   ip6_main_t *im = &ip6_main;
2717   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
2718
2719   ASSERT (option < ARRAY_LEN (hm->options));
2720
2721   /* Not registered */
2722   if (!hm->options[option])
2723     return (-1);
2724
2725   hm->options[option] = NULL;
2726   hm->trace[option] = NULL;
2727
2728   /* Disable global knob if this was the last option configured */
2729   int i;
2730   bool found = false;
2731   for (i = 0; i < 256; i++)
2732     {
2733       if (hm->options[option])
2734         {
2735           found = true;
2736           break;
2737         }
2738     }
2739   if (!found)
2740     im->hbh_enabled = 0;
2741
2742   return (0);
2743 }
2744
2745 /* Global IP6 main. */
2746 ip6_main_t ip6_main;
2747
2748 static clib_error_t *
2749 ip6_lookup_init (vlib_main_t * vm)
2750 {
2751   ip6_main_t *im = &ip6_main;
2752   clib_error_t *error;
2753   uword i;
2754
2755   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
2756     return error;
2757
2758   for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
2759     {
2760       u32 j, i0, i1;
2761
2762       i0 = i / 32;
2763       i1 = i % 32;
2764
2765       for (j = 0; j < i0; j++)
2766         im->fib_masks[i].as_u32[j] = ~0;
2767
2768       if (i1)
2769         im->fib_masks[i].as_u32[i0] =
2770           clib_host_to_net_u32 (pow2_mask (i1) << (32 - i1));
2771     }
2772
2773   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 1);
2774
2775   if (im->lookup_table_nbuckets == 0)
2776     im->lookup_table_nbuckets = IP6_FIB_DEFAULT_HASH_NUM_BUCKETS;
2777
2778   im->lookup_table_nbuckets = 1 << max_log2 (im->lookup_table_nbuckets);
2779
2780   if (im->lookup_table_size == 0)
2781     im->lookup_table_size = IP6_FIB_DEFAULT_HASH_MEMORY_SIZE;
2782
2783   BV (clib_bihash_init) (&(im->ip6_table[IP6_FIB_TABLE_FWDING].ip6_hash),
2784                          "ip6 FIB fwding table",
2785                          im->lookup_table_nbuckets, im->lookup_table_size);
2786   BV (clib_bihash_init) (&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
2787                          "ip6 FIB non-fwding table",
2788                          im->lookup_table_nbuckets, im->lookup_table_size);
2789
2790   /* Create FIB with index 0 and table id of 0. */
2791   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2792                                      FIB_SOURCE_DEFAULT_ROUTE);
2793   mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
2794                                       MFIB_SOURCE_DEFAULT_ROUTE);
2795
2796   {
2797     pg_node_t *pn;
2798     pn = pg_get_node (ip6_lookup_node.index);
2799     pn->unformat_edit = unformat_pg_ip6_header;
2800   }
2801
2802   /* Unless explicitly configured, don't process HBH options */
2803   im->hbh_enabled = 0;
2804
2805   {
2806     icmp6_neighbor_solicitation_header_t p;
2807
2808     memset (&p, 0, sizeof (p));
2809
2810     p.ip.ip_version_traffic_class_and_flow_label =
2811       clib_host_to_net_u32 (0x6 << 28);
2812     p.ip.payload_length =
2813       clib_host_to_net_u16 (sizeof (p) -
2814                             STRUCT_OFFSET_OF
2815                             (icmp6_neighbor_solicitation_header_t, neighbor));
2816     p.ip.protocol = IP_PROTOCOL_ICMP6;
2817     p.ip.hop_limit = 255;
2818     ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);
2819
2820     p.neighbor.icmp.type = ICMP6_neighbor_solicitation;
2821
2822     p.link_layer_option.header.type =
2823       ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
2824     p.link_layer_option.header.n_data_u64s =
2825       sizeof (p.link_layer_option) / sizeof (u64);
2826
2827     vlib_packet_template_init (vm,
2828                                &im->discover_neighbor_packet_template,
2829                                &p, sizeof (p),
2830                                /* alloc chunk size */ 8,
2831                                "ip6 neighbor discovery");
2832   }
2833
2834   return error;
2835 }
2836
2837 VLIB_INIT_FUNCTION (ip6_lookup_init);
2838
2839 void
2840 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t * ip,
2841                                                   u8 * mac)
2842 {
2843   ip->as_u64[0] = clib_host_to_net_u64 (0xFE80000000000000ULL);
2844   /* Invert the "u" bit */
2845   ip->as_u8[8] = mac[0] ^ (1 << 1);
2846   ip->as_u8[9] = mac[1];
2847   ip->as_u8[10] = mac[2];
2848   ip->as_u8[11] = 0xFF;
2849   ip->as_u8[12] = 0xFE;
2850   ip->as_u8[13] = mac[3];
2851   ip->as_u8[14] = mac[4];
2852   ip->as_u8[15] = mac[5];
2853 }
2854
2855 void
2856 ip6_ethernet_mac_address_from_link_local_address (u8 * mac,
2857                                                   ip6_address_t * ip)
2858 {
2859   /* Invert the previously inverted "u" bit */
2860   mac[0] = ip->as_u8[8] ^ (1 << 1);
2861   mac[1] = ip->as_u8[9];
2862   mac[2] = ip->as_u8[10];
2863   mac[3] = ip->as_u8[13];
2864   mac[4] = ip->as_u8[14];
2865   mac[5] = ip->as_u8[15];
2866 }
2867
2868 static clib_error_t *
2869 test_ip6_link_command_fn (vlib_main_t * vm,
2870                           unformat_input_t * input, vlib_cli_command_t * cmd)
2871 {
2872   u8 mac[6];
2873   ip6_address_t _a, *a = &_a;
2874
2875   if (unformat (input, "%U", unformat_ethernet_address, mac))
2876     {
2877       ip6_link_local_address_from_ethernet_mac_address (a, mac);
2878       vlib_cli_output (vm, "Link local address: %U", format_ip6_address, a);
2879       ip6_ethernet_mac_address_from_link_local_address (mac, a);
2880       vlib_cli_output (vm, "Original MAC address: %U",
2881                        format_ethernet_address, mac);
2882     }
2883
2884   return 0;
2885 }
2886
2887 /*?
2888  * This command converts the given MAC Address into an IPv6 link-local
2889  * address.
2890  *
2891  * @cliexpar
2892  * Example of how to create an IPv6 link-local address:
2893  * @cliexstart{test ip6 link 16:d9:e0:91:79:86}
2894  * Link local address: fe80::14d9:e0ff:fe91:7986
2895  * Original MAC address: 16:d9:e0:91:79:86
2896  * @cliexend
2897 ?*/
2898 /* *INDENT-OFF* */
2899 VLIB_CLI_COMMAND (test_link_command, static) =
2900 {
2901   .path = "test ip6 link",
2902   .function = test_ip6_link_command_fn,
2903   .short_help = "test ip6 link <mac-address>",
2904 };
2905 /* *INDENT-ON* */
2906
2907 int
2908 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
2909 {
2910   u32 fib_index;
2911
2912   fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
2913
2914   if (~0 == fib_index)
2915     return VNET_API_ERROR_NO_SUCH_FIB;
2916
2917   fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
2918                                   flow_hash_config);
2919
2920   return 0;
2921 }
2922
2923 static clib_error_t *
2924 set_ip6_flow_hash_command_fn (vlib_main_t * vm,
2925                               unformat_input_t * input,
2926                               vlib_cli_command_t * cmd)
2927 {
2928   int matched = 0;
2929   u32 table_id = 0;
2930   u32 flow_hash_config = 0;
2931   int rv;
2932
2933   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2934     {
2935       if (unformat (input, "table %d", &table_id))
2936         matched = 1;
2937 #define _(a,v) \
2938     else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
2939       foreach_flow_hash_bit
2940 #undef _
2941         else
2942         break;
2943     }
2944
2945   if (matched == 0)
2946     return clib_error_return (0, "unknown input `%U'",
2947                               format_unformat_error, input);
2948
2949   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
2950   switch (rv)
2951     {
2952     case 0:
2953       break;
2954
2955     case -1:
2956       return clib_error_return (0, "no such FIB table %d", table_id);
2957
2958     default:
2959       clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
2960       break;
2961     }
2962
2963   return 0;
2964 }
2965
2966 /*?
2967  * Configure the set of IPv6 fields used by the flow hash.
2968  *
2969  * @cliexpar
2970  * @parblock
2971  * Example of how to set the flow hash on a given table:
2972  * @cliexcmd{set ip6 flow-hash table 8 dst sport dport proto}
2973  *
2974  * Example of display the configured flow hash:
2975  * @cliexstart{show ip6 fib}
2976  * ipv6-VRF:0, fib_index 0, flow hash: src dst sport dport proto
2977  * @::/0
2978  *   unicast-ip6-chain
2979  *   [@0]: dpo-load-balance: [index:5 buckets:1 uRPF:5 to:[0:0]]
2980  *     [0] [@0]: dpo-drop ip6
2981  * fe80::/10
2982  *   unicast-ip6-chain
2983  *   [@0]: dpo-load-balance: [index:10 buckets:1 uRPF:10 to:[0:0]]
2984  *     [0] [@2]: dpo-receive
2985  * ff02::1/128
2986  *   unicast-ip6-chain
2987  *   [@0]: dpo-load-balance: [index:8 buckets:1 uRPF:8 to:[0:0]]
2988  *     [0] [@2]: dpo-receive
2989  * ff02::2/128
2990  *   unicast-ip6-chain
2991  *   [@0]: dpo-load-balance: [index:7 buckets:1 uRPF:7 to:[0:0]]
2992  *     [0] [@2]: dpo-receive
2993  * ff02::16/128
2994  *   unicast-ip6-chain
2995  *   [@0]: dpo-load-balance: [index:9 buckets:1 uRPF:9 to:[0:0]]
2996  *     [0] [@2]: dpo-receive
2997  * ff02::1:ff00:0/104
2998  *   unicast-ip6-chain
2999  *   [@0]: dpo-load-balance: [index:6 buckets:1 uRPF:6 to:[0:0]]
3000  *     [0] [@2]: dpo-receive
3001  * ipv6-VRF:8, fib_index 1, flow hash: dst sport dport proto
3002  * @::/0
3003  *   unicast-ip6-chain
3004  *   [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3005  *     [0] [@0]: dpo-drop ip6
3006  * @::a:1:1:0:4/126
3007  *   unicast-ip6-chain
3008  *   [@0]: dpo-load-balance: [index:27 buckets:1 uRPF:26 to:[0:0]]
3009  *     [0] [@4]: ipv6-glean: af_packet0
3010  * @::a:1:1:0:7/128
3011  *   unicast-ip6-chain
3012  *   [@0]: dpo-load-balance: [index:28 buckets:1 uRPF:27 to:[0:0]]
3013  *     [0] [@2]: dpo-receive: @::a:1:1:0:7 on af_packet0
3014  * fe80::/10
3015  *   unicast-ip6-chain
3016  *   [@0]: dpo-load-balance: [index:26 buckets:1 uRPF:25 to:[0:0]]
3017  *     [0] [@2]: dpo-receive
3018  * fe80::fe:3eff:fe3e:9222/128
3019  *   unicast-ip6-chain
3020  *   [@0]: dpo-load-balance: [index:29 buckets:1 uRPF:28 to:[0:0]]
3021  *     [0] [@2]: dpo-receive: fe80::fe:3eff:fe3e:9222 on af_packet0
3022  * ff02::1/128
3023  *   unicast-ip6-chain
3024  *   [@0]: dpo-load-balance: [index:24 buckets:1 uRPF:23 to:[0:0]]
3025  *     [0] [@2]: dpo-receive
3026  * ff02::2/128
3027  *   unicast-ip6-chain
3028  *   [@0]: dpo-load-balance: [index:23 buckets:1 uRPF:22 to:[0:0]]
3029  *     [0] [@2]: dpo-receive
3030  * ff02::16/128
3031  *   unicast-ip6-chain
3032  *   [@0]: dpo-load-balance: [index:25 buckets:1 uRPF:24 to:[0:0]]
3033  *     [0] [@2]: dpo-receive
3034  * ff02::1:ff00:0/104
3035  *   unicast-ip6-chain
3036  *   [@0]: dpo-load-balance: [index:22 buckets:1 uRPF:21 to:[0:0]]
3037  *     [0] [@2]: dpo-receive
3038  * @cliexend
3039  * @endparblock
3040 ?*/
3041 /* *INDENT-OFF* */
3042 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) =
3043 {
3044   .path = "set ip6 flow-hash",
3045   .short_help =
3046   "set ip6 flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3047   .function = set_ip6_flow_hash_command_fn,
3048 };
3049 /* *INDENT-ON* */
3050
3051 static clib_error_t *
3052 show_ip6_local_command_fn (vlib_main_t * vm,
3053                            unformat_input_t * input, vlib_cli_command_t * cmd)
3054 {
3055   ip6_main_t *im = &ip6_main;
3056   ip_lookup_main_t *lm = &im->lookup_main;
3057   int i;
3058
3059   vlib_cli_output (vm, "Protocols handled by ip6_local");
3060   for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
3061     {
3062       if (lm->local_next_by_ip_protocol[i] != IP_LOCAL_NEXT_PUNT)
3063         {
3064
3065           u32 node_index = vlib_get_node (vm,
3066                                           ip6_local_node.index)->
3067             next_nodes[lm->local_next_by_ip_protocol[i]];
3068           vlib_cli_output (vm, "%d: %U", i, format_vlib_node_name, vm,
3069                            node_index);
3070         }
3071     }
3072   return 0;
3073 }
3074
3075
3076
3077 /*?
3078  * Display the set of protocols handled by the local IPv6 stack.
3079  *
3080  * @cliexpar
3081  * Example of how to display local protocol table:
3082  * @cliexstart{show ip6 local}
3083  * Protocols handled by ip6_local
3084  * 17
3085  * 43
3086  * 58
3087  * 115
3088  * @cliexend
3089 ?*/
3090 /* *INDENT-OFF* */
3091 VLIB_CLI_COMMAND (show_ip6_local, static) =
3092 {
3093   .path = "show ip6 local",
3094   .function = show_ip6_local_command_fn,
3095   .short_help = "show ip6 local",
3096 };
3097 /* *INDENT-ON* */
3098
3099 int
3100 vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
3101                              u32 table_index)
3102 {
3103   vnet_main_t *vnm = vnet_get_main ();
3104   vnet_interface_main_t *im = &vnm->interface_main;
3105   ip6_main_t *ipm = &ip6_main;
3106   ip_lookup_main_t *lm = &ipm->lookup_main;
3107   vnet_classify_main_t *cm = &vnet_classify_main;
3108   ip6_address_t *if_addr;
3109
3110   if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3111     return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3112
3113   if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3114     return VNET_API_ERROR_NO_SUCH_ENTRY;
3115
3116   vec_validate (lm->classify_table_index_by_sw_if_index, sw_if_index);
3117   lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3118
3119   if_addr = ip6_interface_first_address (ipm, sw_if_index);
3120
3121   if (NULL != if_addr)
3122     {
3123       fib_prefix_t pfx = {
3124         .fp_len = 128,
3125         .fp_proto = FIB_PROTOCOL_IP6,
3126         .fp_addr.ip6 = *if_addr,
3127       };
3128       u32 fib_index;
3129
3130       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
3131                                                        sw_if_index);
3132
3133
3134       if (table_index != (u32) ~ 0)
3135         {
3136           dpo_id_t dpo = DPO_INVALID;
3137
3138           dpo_set (&dpo,
3139                    DPO_CLASSIFY,
3140                    DPO_PROTO_IP6,
3141                    classify_dpo_create (DPO_PROTO_IP6, table_index));
3142
3143           fib_table_entry_special_dpo_add (fib_index,
3144                                            &pfx,
3145                                            FIB_SOURCE_CLASSIFY,
3146                                            FIB_ENTRY_FLAG_NONE, &dpo);
3147           dpo_reset (&dpo);
3148         }
3149       else
3150         {
3151           fib_table_entry_special_remove (fib_index,
3152                                           &pfx, FIB_SOURCE_CLASSIFY);
3153         }
3154     }
3155
3156   return 0;
3157 }
3158
3159 static clib_error_t *
3160 set_ip6_classify_command_fn (vlib_main_t * vm,
3161                              unformat_input_t * input,
3162                              vlib_cli_command_t * cmd)
3163 {
3164   u32 table_index = ~0;
3165   int table_index_set = 0;
3166   u32 sw_if_index = ~0;
3167   int rv;
3168
3169   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3170     {
3171       if (unformat (input, "table-index %d", &table_index))
3172         table_index_set = 1;
3173       else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3174                          vnet_get_main (), &sw_if_index))
3175         ;
3176       else
3177         break;
3178     }
3179
3180   if (table_index_set == 0)
3181     return clib_error_return (0, "classify table-index must be specified");
3182
3183   if (sw_if_index == ~0)
3184     return clib_error_return (0, "interface / subif must be specified");
3185
3186   rv = vnet_set_ip6_classify_intfc (vm, sw_if_index, table_index);
3187
3188   switch (rv)
3189     {
3190     case 0:
3191       break;
3192
3193     case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3194       return clib_error_return (0, "No such interface");
3195
3196     case VNET_API_ERROR_NO_SUCH_ENTRY:
3197       return clib_error_return (0, "No such classifier table");
3198     }
3199   return 0;
3200 }
3201
3202 /*?
3203  * Assign a classification table to an interface. The classification
3204  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3205  * commands. Once the table is create, use this command to filter packets
3206  * on an interface.
3207  *
3208  * @cliexpar
3209  * Example of how to assign a classification table to an interface:
3210  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
3211 ?*/
3212 /* *INDENT-OFF* */
3213 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
3214 {
3215   .path = "set ip6 classify",
3216   .short_help =
3217   "set ip6 classify intfc <interface> table-index <classify-idx>",
3218   .function = set_ip6_classify_command_fn,
3219 };
3220 /* *INDENT-ON* */
3221
3222 static clib_error_t *
3223 ip6_config (vlib_main_t * vm, unformat_input_t * input)
3224 {
3225   ip6_main_t *im = &ip6_main;
3226   uword heapsize = 0;
3227   u32 tmp;
3228   u32 nbuckets = 0;
3229
3230   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3231     {
3232       if (unformat (input, "hash-buckets %d", &tmp))
3233         nbuckets = tmp;
3234       else if (unformat (input, "heap-size %U",
3235                          unformat_memory_size, &heapsize))
3236         ;
3237       else
3238         return clib_error_return (0, "unknown input '%U'",
3239                                   format_unformat_error, input);
3240     }
3241
3242   im->lookup_table_nbuckets = nbuckets;
3243   im->lookup_table_size = heapsize;
3244
3245   return 0;
3246 }
3247
3248 VLIB_EARLY_CONFIG_FUNCTION (ip6_config, "ip6");
3249
3250 /*
3251  * fd.io coding-style-patch-verification: ON
3252  *
3253  * Local Variables:
3254  * eval: (c-set-style "gnu")
3255  * End:
3256  */