IPv6 Performance bugs
[vpp.git] / src / vnet / ip / ip6_forward.c
index 2388a30..28c84d1 100644 (file)
@@ -45,7 +45,7 @@
 #include <vnet/fib/fib_urpf_list.h>    /* for FIB uRPF check */
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/mfib/ip6_mfib.h>
-#include <vnet/dpo/load_balance.h>
+#include <vnet/dpo/load_balance_map.h>
 #include <vnet/dpo/classify_dpo.h>
 
 #include <vppinfra/bihash_template.c>
@@ -74,7 +74,7 @@ ip6_lookup_inline (vlib_main_t * vm,
   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
   u32 n_left_from, n_left_to_next, *from, *to_next;
   ip_lookup_next_t next;
-  u32 cpu_index = os_get_cpu_number ();
+  u32 thread_index = vlib_get_thread_index ();
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
@@ -138,6 +138,10 @@ ip6_lookup_inline (vlib_main_t * vm,
 
          lb0 = load_balance_get (lbi0);
          lb1 = load_balance_get (lbi1);
+         ASSERT (lb0->lb_n_buckets > 0);
+         ASSERT (lb1->lb_n_buckets > 0);
+         ASSERT (is_pow2 (lb0->lb_n_buckets));
+         ASSERT (is_pow2 (lb1->lb_n_buckets));
 
          vnet_buffer (p0)->ip.flow_hash = vnet_buffer (p1)->ip.flow_hash = 0;
 
@@ -146,25 +150,29 @@ ip6_lookup_inline (vlib_main_t * vm,
              flow_hash_config0 = lb0->lb_hash_config;
              vnet_buffer (p0)->ip.flow_hash =
                ip6_compute_flow_hash (ip0, flow_hash_config0);
+             dpo0 =
+               load_balance_get_fwd_bucket (lb0,
+                                            (vnet_buffer (p0)->ip.flow_hash &
+                                             (lb0->lb_n_buckets_minus_1)));
+           }
+         else
+           {
+             dpo0 = load_balance_get_bucket_i (lb0, 0);
            }
          if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
            {
              flow_hash_config1 = lb1->lb_hash_config;
              vnet_buffer (p1)->ip.flow_hash =
                ip6_compute_flow_hash (ip1, flow_hash_config1);
+             dpo1 =
+               load_balance_get_fwd_bucket (lb1,
+                                            (vnet_buffer (p1)->ip.flow_hash &
+                                             (lb1->lb_n_buckets_minus_1)));
+           }
+         else
+           {
+             dpo1 = load_balance_get_bucket_i (lb1, 0);
            }
-
-         ASSERT (lb0->lb_n_buckets > 0);
-         ASSERT (lb1->lb_n_buckets > 0);
-         ASSERT (is_pow2 (lb0->lb_n_buckets));
-         ASSERT (is_pow2 (lb1->lb_n_buckets));
-         dpo0 = load_balance_get_bucket_i (lb0,
-                                           (vnet_buffer (p0)->ip.flow_hash &
-                                            lb0->lb_n_buckets_minus_1));
-         dpo1 = load_balance_get_bucket_i (lb1,
-                                           (vnet_buffer (p1)->ip.flow_hash &
-                                            lb1->lb_n_buckets_minus_1));
-
          next0 = dpo0->dpoi_next_node;
          next1 = dpo1->dpoi_next_node;
 
@@ -185,9 +193,9 @@ ip6_lookup_inline (vlib_main_t * vm,
          vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
+           (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
+           (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 
          from += 2;
          to_next += 2;
@@ -259,23 +267,30 @@ ip6_lookup_inline (vlib_main_t * vm,
            (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
             (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 
-         flow_hash_config0 = ip6_fib_get (fib_index0)->flow_hash_config;
-
          lbi0 = ip6_fib_table_fwding_lookup (im, fib_index0, dst_addr0);
 
          lb0 = load_balance_get (lbi0);
+         flow_hash_config0 = lb0->lb_hash_config;
 
          vnet_buffer (p0)->ip.flow_hash = 0;
+         ASSERT (lb0->lb_n_buckets > 0);
+         ASSERT (is_pow2 (lb0->lb_n_buckets));
 
          if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
            {
              flow_hash_config0 = lb0->lb_hash_config;
              vnet_buffer (p0)->ip.flow_hash =
                ip6_compute_flow_hash (ip0, flow_hash_config0);
+             dpo0 =
+               load_balance_get_fwd_bucket (lb0,
+                                            (vnet_buffer (p0)->ip.flow_hash &
+                                             (lb0->lb_n_buckets_minus_1)));
+           }
+         else
+           {
+             dpo0 = load_balance_get_bucket_i (lb0, 0);
            }
 
-         ASSERT (lb0->lb_n_buckets > 0);
-         ASSERT (is_pow2 (lb0->lb_n_buckets));
          dpo0 = load_balance_get_bucket_i (lb0,
                                            (vnet_buffer (p0)->ip.flow_hash &
                                             lb0->lb_n_buckets_minus_1));
@@ -291,7 +306,7 @@ ip6_lookup_inline (vlib_main_t * vm,
          vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
 
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
+           (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 
          from += 1;
          to_next += 1;
@@ -337,10 +352,18 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
     {
       fib_node_index_t fei;
 
-      fei = fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_ATTACHED), FIB_PROTOCOL_IP6, NULL,      /* No next-hop address */
-                                            sw_if_index, ~0,   // invalid FIB index
-                                            1, NULL,   // no label stack
-                                            FIB_ROUTE_PATH_FLAG_NONE);
+      fei = fib_table_entry_update_one_path (fib_index,
+                                            &pfx,
+                                            FIB_SOURCE_INTERFACE,
+                                            (FIB_ENTRY_FLAG_CONNECTED |
+                                             FIB_ENTRY_FLAG_ATTACHED),
+                                            FIB_PROTOCOL_IP6,
+                                            /* No next-hop address */
+                                            NULL, sw_if_index,
+                                            /* invalid FIB index */
+                                            ~0, 1,
+                                            /* no label stack */
+                                            NULL, FIB_ROUTE_PATH_FLAG_NONE);
       a->neighbor_probe_adj_index = fib_entry_get_adj (fei);
     }
 
@@ -366,7 +389,13 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
        }
     }
 
-  fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL), FIB_PROTOCOL_IP6, &pfx.fp_addr, sw_if_index, ~0,  // invalid FIB index
+  fib_table_entry_update_one_path (fib_index, &pfx,
+                                  FIB_SOURCE_INTERFACE,
+                                  (FIB_ENTRY_FLAG_CONNECTED |
+                                   FIB_ENTRY_FLAG_LOCAL),
+                                  FIB_PROTOCOL_IP6,
+                                  &pfx.fp_addr,
+                                  sw_if_index, ~0,
                                   1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
 }
 
@@ -415,12 +444,11 @@ ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
        return;
     }
 
-  vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index,
-                              is_enable, 0, 0);
-
-  vnet_feature_enable_disable ("ip6-multicast", "ip6-mfib-forward-lookup",
-                              sw_if_index, is_enable, 0, 0);
+  vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index,
+                              !is_enable, 0, 0);
 
+  vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index,
+                              !is_enable, 0, 0);
 }
 
 /* get first interface address */
@@ -595,17 +623,17 @@ VNET_FEATURE_INIT (ip6_vxlan_bypass, static) =
   .runs_before = VNET_FEATURES ("ip6-lookup"),
 };
 
-VNET_FEATURE_INIT (ip6_lookup, static) =
+VNET_FEATURE_INIT (ip6_drop, static) =
 {
   .arc_name = "ip6-unicast",
-  .node_name = "ip6-lookup",
-  .runs_before = VNET_FEATURES ("ip6-drop"),
+  .node_name = "ip6-drop",
+  .runs_before = VNET_FEATURES ("ip6-lookup"),
 };
 
-VNET_FEATURE_INIT (ip6_drop, static) =
+VNET_FEATURE_INIT (ip6_lookup, static) =
 {
   .arc_name = "ip6-unicast",
-  .node_name = "ip6-drop",
+  .node_name = "ip6-lookup",
   .runs_before = 0,  /*last feature*/
 };
 
@@ -623,15 +651,15 @@ VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
   .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 };
 
-VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
+VNET_FEATURE_INIT (ip6_drop_mc, static) = {
   .arc_name = "ip6-multicast",
-  .node_name = "ip6-mfib-forward-lookup",
-  .runs_before = VNET_FEATURES ("ip6-drop"),
+  .node_name = "ip6-drop",
+  .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 };
 
-VNET_FEATURE_INIT (ip6_drop_mc, static) = {
+VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
   .arc_name = "ip6-multicast",
-  .node_name = "ip6-drop",
+  .node_name = "ip6-mfib-forward-lookup",
   .runs_before = 0, /* last feature */
 };
 
@@ -659,15 +687,17 @@ VNET_FEATURE_INIT (ip6_interface_output, static) = {
 clib_error_t *
 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 {
+  ip6_main_t *im = &ip6_main;
+
+  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
+  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
+
   vnet_feature_enable_disable ("ip6-unicast", "ip6-drop", sw_if_index,
                               is_add, 0, 0);
 
   vnet_feature_enable_disable ("ip6-multicast", "ip6-drop", sw_if_index,
                               is_add, 0, 0);
 
-  vnet_feature_enable_disable ("ip6-output", "interface-output", sw_if_index,
-                              is_add, 0, 0);
-
   return /* no error */ 0;
 }
 
@@ -703,7 +733,7 @@ ip6_load_balance (vlib_main_t * vm,
   vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters;
   u32 n_left_from, n_left_to_next, *from, *to_next;
   ip_lookup_next_t next;
-  u32 cpu_index = os_get_cpu_number ();
+  u32 thread_index = vlib_get_thread_index ();
   ip6_main_t *im = &ip6_main;
 
   from = vlib_frame_vector_args (frame);
@@ -766,8 +796,7 @@ ip6_load_balance (vlib_main_t * vm,
           * We don't want to use the same hash value at each level in the recursion
           * graph as that would lead to polarisation
           */
-         hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
-         hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
+         hc0 = hc1 = 0;
 
          if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
            {
@@ -779,8 +808,16 @@ ip6_load_balance (vlib_main_t * vm,
              else
                {
                  hc0 = vnet_buffer (p0)->ip.flow_hash =
-                   ip6_compute_flow_hash (ip0, hc0);
+                   ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
                }
+             dpo0 =
+               load_balance_get_fwd_bucket (lb0,
+                                            (hc0 &
+                                             lb0->lb_n_buckets_minus_1));
+           }
+         else
+           {
+             dpo0 = load_balance_get_bucket_i (lb0, 0);
            }
          if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
            {
@@ -792,16 +829,17 @@ ip6_load_balance (vlib_main_t * vm,
              else
                {
                  hc1 = vnet_buffer (p1)->ip.flow_hash =
-                   ip6_compute_flow_hash (ip1, hc1);
+                   ip6_compute_flow_hash (ip1, lb1->lb_hash_config);
                }
+             dpo1 =
+               load_balance_get_fwd_bucket (lb1,
+                                            (hc1 &
+                                             lb1->lb_n_buckets_minus_1));
+           }
+         else
+           {
+             dpo1 = load_balance_get_bucket_i (lb1, 0);
            }
-
-         dpo0 =
-           load_balance_get_bucket_i (lb0,
-                                      hc0 & (lb0->lb_n_buckets_minus_1));
-         dpo1 =
-           load_balance_get_bucket_i (lb1,
-                                      hc1 & (lb1->lb_n_buckets_minus_1));
 
          next0 = dpo0->dpoi_next_node;
          next1 = dpo1->dpoi_next_node;
@@ -825,9 +863,9 @@ ip6_load_balance (vlib_main_t * vm,
          vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
 
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
+           (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
+           (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
 
          vlib_validate_buffer_enqueue_x2 (vm, node, next,
                                           to_next, n_left_to_next,
@@ -857,7 +895,7 @@ ip6_load_balance (vlib_main_t * vm,
 
          lb0 = load_balance_get (lbi0);
 
-         hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
+         hc0 = 0;
          if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
            {
              if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
@@ -868,12 +906,17 @@ ip6_load_balance (vlib_main_t * vm,
              else
                {
                  hc0 = vnet_buffer (p0)->ip.flow_hash =
-                   ip6_compute_flow_hash (ip0, hc0);
+                   ip6_compute_flow_hash (ip0, lb0->lb_hash_config);
                }
+             dpo0 =
+               load_balance_get_fwd_bucket (lb0,
+                                            (hc0 &
+                                             lb0->lb_n_buckets_minus_1));
+           }
+         else
+           {
+             dpo0 = load_balance_get_bucket_i (lb0, 0);
            }
-         dpo0 =
-           load_balance_get_bucket_i (lb0,
-                                      hc0 & (lb0->lb_n_buckets_minus_1));
 
          next0 = dpo0->dpoi_next_node;
          vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
@@ -887,7 +930,7 @@ ip6_load_balance (vlib_main_t * vm,
            }
 
          vlib_increment_combined_counter
-           (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
+           (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
 
          vlib_validate_buffer_enqueue_x1 (vm, node, next,
                                           to_next, n_left_to_next,
@@ -962,7 +1005,6 @@ format_ip6_rewrite_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
-  vnet_main_t *vnm = vnet_get_main ();
   uword indent = format_get_indent (s);
 
   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
@@ -971,7 +1013,7 @@ format_ip6_rewrite_trace (u8 * s, va_list * args)
   s = format (s, "\n%U%U",
              format_white_space, indent,
              format_ip_adjacency_packet_data,
-             vnm, t->adj_index, t->packet_data, sizeof (t->packet_data));
+             t->adj_index, t->packet_data, sizeof (t->packet_data));
   return s;
 }
 
@@ -1162,7 +1204,8 @@ ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
                                                uword));
     }
 
-  /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets) */
+  /* some icmp packets may come with a "router alert" hop-by-hop extension header (e.g., mldv2 packets)
+   * or UDP-Ping packets */
   if (PREDICT_FALSE (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
     {
       u32 skip_bytes;
@@ -1170,7 +1213,8 @@ ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
        (ip6_hop_by_hop_ext_t *) data_this_buffer;
 
       /* validate really icmp6 next */
-      ASSERT (ext_hdr->next_hdr == IP_PROTOCOL_ICMP6);
+      ASSERT ((ext_hdr->next_hdr == IP_PROTOCOL_ICMP6)
+             || (ext_hdr->next_hdr == IP_PROTOCOL_UDP));
 
       skip_bytes = 8 * (1 + ext_hdr->n_data_u64s);
       data_this_buffer = (void *) ((u8 *) data_this_buffer + skip_bytes);
@@ -1318,23 +1362,6 @@ ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
          len_diff0 = 0;
          len_diff1 = 0;
 
-         /* Skip HBH local processing */
-         if (PREDICT_FALSE
-             (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-           {
-             ip6_hop_by_hop_ext_t *ext_hdr =
-               (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-             next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
-             type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
-           }
-         if (PREDICT_FALSE
-             (ip1->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-           {
-             ip6_hop_by_hop_ext_t *ext_hdr =
-               (ip6_hop_by_hop_ext_t *) ip6_next_header (ip1);
-             next1 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
-             type1 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
-           }
          if (PREDICT_TRUE (IP_PROTOCOL_UDP == ip6_locate_header (p0, ip0,
                                                                  IP_PROTOCOL_UDP,
                                                                  &udp_offset0)))
@@ -1459,15 +1486,6 @@ ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
          good_l4_checksum0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
          len_diff0 = 0;
 
-         /* Skip HBH local processing */
-         if (PREDICT_FALSE
-             (ip0->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS))
-           {
-             ip6_hop_by_hop_ext_t *ext_hdr =
-               (ip6_hop_by_hop_ext_t *) ip6_next_header (ip0);
-             next0 = lm->local_next_by_ip_protocol[ext_hdr->next_hdr];
-             type0 = lm->builtin_protocol_by_ip_protocol[ext_hdr->next_hdr];
-           }
          if (PREDICT_TRUE (IP_PROTOCOL_UDP == ip6_locate_header (p0, ip0,
                                                                  IP_PROTOCOL_UDP,
                                                                  &udp_offset0)))
@@ -1637,7 +1655,7 @@ ip6_discover_neighbor_inline (vlib_main_t * vm,
 
          ip0 = vlib_buffer_get_current (p0);
 
-         adj0 = ip_get_adjacency (lm, adj_index0);
+         adj0 = adj_get (adj_index0);
 
          if (!is_glean)
            {
@@ -1888,7 +1906,7 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
 
   /* Add encapsulation string for software interface (e.g. ethernet header). */
-  adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
+  adj = adj_get (ia->neighbor_probe_adj_index);
   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
 
@@ -1912,7 +1930,8 @@ typedef enum
 always_inline uword
 ip6_rewrite_inline (vlib_main_t * vm,
                    vlib_node_runtime_t * node,
-                   vlib_frame_t * frame, int is_midchain, int is_mcast)
+                   vlib_frame_t * frame,
+                   int do_counters, int is_midchain, int is_mcast)
 {
   ip_lookup_main_t *lm = &ip6_main.lookup_main;
   u32 *from = vlib_frame_vector_args (frame);
@@ -1922,7 +1941,7 @@ ip6_rewrite_inline (vlib_main_t * vm,
 
   n_left_from = frame->n_vectors;
   next_index = node->cached_next_index;
-  u32 cpu_index = os_get_cpu_number ();
+  u32 thread_index = vlib_get_thread_index ();
 
   while (n_left_from > 0)
     {
@@ -1968,9 +1987,6 @@ ip6_rewrite_inline (vlib_main_t * vm,
          adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
          adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
 
-         /* We should never rewrite a pkt using the MISS adjacency */
-         ASSERT (adj_index0 && adj_index1);
-
          ip0 = vlib_buffer_get_current (p0);
          ip1 = vlib_buffer_get_current (p1);
 
@@ -2035,22 +2051,25 @@ ip6_rewrite_inline (vlib_main_t * vm,
            {
              p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
            }
-         adj0 = ip_get_adjacency (lm, adj_index0);
-         adj1 = ip_get_adjacency (lm, adj_index1);
+         adj0 = adj_get (adj_index0);
+         adj1 = adj_get (adj_index1);
 
          rw_len0 = adj0[0].rewrite_header.data_bytes;
          rw_len1 = adj1[0].rewrite_header.data_bytes;
          vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
          vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
 
-         vlib_increment_combined_counter
-           (&adjacency_counters,
-            cpu_index,
-            adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0);
-         vlib_increment_combined_counter
-           (&adjacency_counters,
-            cpu_index, adj_index1,
-            1, vlib_buffer_length_in_chain (vm, p1) + rw_len1);
+         if (do_counters)
+           {
+             vlib_increment_combined_counter
+               (&adjacency_counters,
+                thread_index, adj_index0, 1,
+                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
+             vlib_increment_combined_counter
+               (&adjacency_counters,
+                thread_index, adj_index1, 1,
+                vlib_buffer_length_in_chain (vm, p1) + rw_len1);
+           }
 
          /* Check MTU of outgoing interface. */
          error0 =
@@ -2075,8 +2094,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
              vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
              next0 = adj0[0].rewrite_header.next_index;
 
-             vnet_feature_arc_start (lm->output_feature_arc_index,
-                                     tx_sw_if_index0, &next0, p0);
+             if (PREDICT_FALSE
+                 (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+               vnet_feature_arc_start (lm->output_feature_arc_index,
+                                       tx_sw_if_index0, &next0, p0);
            }
          if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
            {
@@ -2087,8 +2108,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
              vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
              next1 = adj1[0].rewrite_header.next_index;
 
-             vnet_feature_arc_start (lm->output_feature_arc_index,
-                                     tx_sw_if_index1, &next1, p1);
+             if (PREDICT_FALSE
+                 (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+               vnet_feature_arc_start (lm->output_feature_arc_index,
+                                       tx_sw_if_index1, &next1, p1);
            }
 
          /* Guess we are only writing on simple Ethernet header. */
@@ -2105,8 +2128,8 @@ ip6_rewrite_inline (vlib_main_t * vm,
              /*
               * copy bytes from the IP address into the MAC rewrite
               */
-             vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 0);
-             vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1, 0);
+             vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
+             vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1);
            }
 
          vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
@@ -2129,10 +2152,7 @@ ip6_rewrite_inline (vlib_main_t * vm,
 
          adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 
-         /* We should never rewrite a pkt using the MISS adjacency */
-         ASSERT (adj_index0);
-
-         adj0 = ip_get_adjacency (lm, adj_index0);
+         adj0 = adj_get (adj_index0);
 
          ip0 = vlib_buffer_get_current (p0);
 
@@ -2176,10 +2196,13 @@ ip6_rewrite_inline (vlib_main_t * vm,
          rw_len0 = adj0[0].rewrite_header.data_bytes;
          vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
 
-         vlib_increment_combined_counter
-           (&adjacency_counters,
-            cpu_index,
-            adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0);
+         if (do_counters)
+           {
+             vlib_increment_combined_counter
+               (&adjacency_counters,
+                thread_index, adj_index0, 1,
+                vlib_buffer_length_in_chain (vm, p0) + rw_len0);
+           }
 
          /* Check MTU of outgoing interface. */
          error0 =
@@ -2200,8 +2223,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
              vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
              next0 = adj0[0].rewrite_header.next_index;
 
-             vnet_feature_arc_start (lm->output_feature_arc_index,
-                                     tx_sw_if_index0, &next0, p0);
+             if (PREDICT_FALSE
+                 (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+               vnet_feature_arc_start (lm->output_feature_arc_index,
+                                       tx_sw_if_index0, &next0, p0);
            }
 
          if (is_midchain)
@@ -2210,7 +2235,7 @@ ip6_rewrite_inline (vlib_main_t * vm,
            }
          if (is_mcast)
            {
-             vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 0);
+             vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0);
            }
 
          p0->error = error_node->errors[error0];
@@ -2239,21 +2264,40 @@ static uword
 ip6_rewrite (vlib_main_t * vm,
             vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip6_rewrite_inline (vm, node, frame, 0, 0);
+  if (adj_are_counters_enabled ())
+    return ip6_rewrite_inline (vm, node, frame, 1, 0, 0);
+  else
+    return ip6_rewrite_inline (vm, node, frame, 0, 0, 0);
 }
 
 static uword
 ip6_rewrite_mcast (vlib_main_t * vm,
                   vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip6_rewrite_inline (vm, node, frame, 0, 1);
+  if (adj_are_counters_enabled ())
+    return ip6_rewrite_inline (vm, node, frame, 1, 0, 1);
+  else
+    return ip6_rewrite_inline (vm, node, frame, 0, 0, 1);
 }
 
 static uword
 ip6_midchain (vlib_main_t * vm,
              vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip6_rewrite_inline (vm, node, frame, 1, 0);
+  if (adj_are_counters_enabled ())
+    return ip6_rewrite_inline (vm, node, frame, 1, 1, 0);
+  else
+    return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
+}
+
+static uword
+ip6_mcast_midchain (vlib_main_t * vm,
+                   vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (adj_are_counters_enabled ())
+    return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
+  else
+    return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
 }
 
 /* *INDENT-OFF* */
@@ -2300,6 +2344,19 @@ VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast);
 
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) =
+{
+  .function = ip6_mcast_midchain,
+  .name = "ip6-mcast-midchain",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip6_rewrite_trace,
+  .sibling_of = "ip6-rewrite",
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain);
+
 /*
  * Hop-by-Hop handling
  */
@@ -2518,8 +2575,6 @@ ip6_hop_by_hop (vlib_main_t * vm,
   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
   u32 n_left_from, *from, *to_next;
   ip_lookup_next_t next_index;
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
@@ -2568,9 +2623,9 @@ ip6_hop_by_hop (vlib_main_t * vm,
 
          /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
          u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-         ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0);
+         ip_adjacency_t *adj0 = adj_get (adj_index0);
          u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
-         ip_adjacency_t *adj1 = ip_get_adjacency (lm, adj_index1);
+         ip_adjacency_t *adj1 = adj_get (adj_index1);
 
          /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
          next0 = adj0->lookup_next_index;
@@ -2691,7 +2746,7 @@ ip6_hop_by_hop (vlib_main_t * vm,
           * A HBH option rarely redirects to a different node
           */
          u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-         ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0);
+         ip_adjacency_t *adj0 = adj_get (adj_index0);
          next0 = adj0->lookup_next_index;
 
          ip0 = vlib_buffer_get_current (b0);
@@ -3101,17 +3156,17 @@ VLIB_CLI_COMMAND (test_link_command, static) =
 int
 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
 {
-  ip6_main_t *im6 = &ip6_main;
-  ip6_fib_t *fib;
-  uword *p = hash_get (im6->fib_index_by_table_id, table_id);
+  u32 fib_index;
 
-  if (p == 0)
-    return -1;
+  fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
 
-  fib = ip6_fib_get (p[0]);
+  if (~0 == fib_index)
+    return VNET_API_ERROR_NO_SUCH_FIB;
 
-  fib->flow_hash_config = flow_hash_config;
-  return 1;
+  fib_table_set_flow_hash_config (fib_index, FIB_PROTOCOL_IP6,
+                                 flow_hash_config);
+
+  return 0;
 }
 
 static clib_error_t *
@@ -3143,7 +3198,7 @@ set_ip6_flow_hash_command_fn (vlib_main_t * vm,
   rv = vnet_set_ip6_flow_hash (table_id, flow_hash_config);
   switch (rv)
     {
-    case 1:
+    case 0:
       break;
 
     case -1: