ip: add support for buffer offload metadata in ip midchain
[vpp.git] / src / vnet / ip / ip6_forward.c
index 0ed20ee..06c473b 100644 (file)
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/mfib/ip6_mfib.h>
 #include <vnet/dpo/load_balance_map.h>
+#include <vnet/dpo/receive_dpo.h>
 #include <vnet/dpo/classify_dpo.h>
 #include <vnet/classify/vnet_classify.h>
+#include <vnet/pg/pg.h>
 
 #ifndef CLIB_MARCH_VARIANT
 #include <vppinfra/bihash_template.c>
 #endif
 #include <vnet/ip/ip6_forward.h>
-#include <vnet/ipsec/ipsec_tun.h>
 #include <vnet/interface_output.h>
 
 /* Flag used by IOAM code. Classifier sets it pop-hop-by-hop checks it */
@@ -70,7 +71,6 @@ ip6_add_interface_prefix_routes (ip6_main_t * im,
   ip_lookup_main_t *lm = &im->lookup_main;
   ip_interface_prefix_t *if_prefix;
 
-  /* *INDENT-OFF* */
   ip_interface_prefix_key_t key = {
     .prefix = {
       .fp_len = address_length,
@@ -84,7 +84,6 @@ ip6_add_interface_prefix_routes (ip6_main_t * im,
     },
     .sw_if_index = sw_if_index,
   };
-  /* *INDENT-ON* */
 
   /* If prefix already set on interface, just increment ref count & return */
   if_prefix = ip_get_interface_prefix (lm, &key);
@@ -177,7 +176,6 @@ ip6_del_interface_prefix_routes (ip6_main_t * im,
   ip_lookup_main_t *lm = &im->lookup_main;
   ip_interface_prefix_t *if_prefix;
 
-  /* *INDENT-OFF* */
   ip_interface_prefix_key_t key = {
     .prefix = {
       .fp_len = address_length,
@@ -191,13 +189,12 @@ ip6_del_interface_prefix_routes (ip6_main_t * im,
     },
     .sw_if_index = sw_if_index,
   };
-  /* *INDENT-ON* */
 
   if_prefix = ip_get_interface_prefix (lm, &key);
   if (!if_prefix)
     {
       clib_warning ("Prefix not found while deleting %U",
-                   format_ip4_address_and_length, address, address_length);
+                   format_ip6_address_and_length, address, address_length);
       return;
     }
 
@@ -282,7 +279,6 @@ ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
   ip_interface_address_t *ia = 0;
   ip6_address_t *result = 0;
 
-  /* *INDENT-OFF* */
   foreach_ip_interface_address (lm, ia, sw_if_index,
                                 1 /* honor unnumbered */,
   ({
@@ -290,7 +286,6 @@ ip6_interface_first_address (ip6_main_t * im, u32 sw_if_index)
     result = a;
     break;
   }));
-  /* *INDENT-ON* */
   return result;
 }
 
@@ -308,11 +303,11 @@ ip6_add_del_interface_address (vlib_main_t * vm,
   ip6_address_fib_t ip6_af, *addr_fib = 0;
   const ip6_address_t *ll_addr;
 
-  /* local0 interface doesn't support IP addressing */
-  if (sw_if_index == 0)
+  error = vnet_sw_interface_supports_addressing (vnm, sw_if_index);
+  if (error)
     {
-      return
-       clib_error_create ("local0 interface doesn't support IP addressing");
+      vnm->api_errno = VNET_API_ERROR_UNSUPPORTED;
+      return error;
     }
 
   if (ip6_address_is_link_local_unicast (address))
@@ -354,14 +349,10 @@ ip6_add_del_interface_address (vlib_main_t * vm,
       return (NULL);
     }
 
-  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
-  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
-
   ip6_addr_fib_init (&ip6_af, address,
                     vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
   vec_add1 (addr_fib, ip6_af);
 
-  /* *INDENT-OFF* */
   if (!is_del)
     {
       /* When adding an address check that it does not conflict
@@ -419,7 +410,6 @@ ip6_add_del_interface_address (vlib_main_t * vm,
             }
       }
     }
-  /* *INDENT-ON* */
 
   if_address_index = ip_interface_address_find (lm, addr_fib, address_length);
 
@@ -531,9 +521,6 @@ ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
   ip6_address_t *a;
   u32 is_admin_up, fib_index;
 
-  /* Fill in lookup tables with default table (0). */
-  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
-
   vec_validate_init_empty (im->
                           lookup_main.if_address_pool_index_by_sw_if_index,
                           sw_if_index, ~0);
@@ -542,7 +529,6 @@ ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 
   fib_index = vec_elt (im->fib_index_by_sw_if_index, sw_if_index);
 
-  /* *INDENT-OFF* */
   foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
                                 0 /* honor unnumbered */,
   ({
@@ -555,7 +541,6 @@ ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
       ip6_del_interface_routes (sw_if_index, im, fib_index,
                                a, ia->address_length);
   }));
-  /* *INDENT-ON* */
 
   return 0;
 }
@@ -563,7 +548,6 @@ ip6_sw_interface_admin_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags)
 VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_sw_interface_admin_up_down);
 
 /* Built-in ip6 unicast rx feature path definition */
-/* *INDENT-OFF* */
 VNET_FEATURE_ARC_INIT (ip6_unicast, static) =
 {
   .arc_name  = "ip6-unicast",
@@ -688,17 +672,22 @@ VNET_FEATURE_INIT (ip6_interface_output, static) = {
   .node_name = "interface-output",
   .runs_before = 0, /* not before any other features */
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 {
   ip6_main_t *im = &ip6_main;
 
-  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
-  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
+  vec_validate_init_empty (im->fib_index_by_sw_if_index, sw_if_index, ~0);
+  vec_validate_init_empty (im->mfib_index_by_sw_if_index, sw_if_index, ~0);
 
-  if (!is_add)
+  if (is_add)
+    {
+      /* Fill in lookup tables with default table (0). */
+      im->fib_index_by_sw_if_index[sw_if_index] = 0;
+      im->mfib_index_by_sw_if_index[sw_if_index] = 0;
+    }
+  else
     {
       /* Ensure that IPv6 is disabled */
       ip6_main_t *im6 = &ip6_main;
@@ -708,14 +697,21 @@ ip6_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
       vlib_main_t *vm = vlib_get_main ();
 
       vnet_sw_interface_update_unnumbered (sw_if_index, ~0, 0);
-      /* *INDENT-OFF* */
       foreach_ip_interface_address (lm6, ia, sw_if_index, 0,
       ({
         address = ip_interface_address_get_address (lm6, ia);
         ip6_add_del_interface_address(vm, sw_if_index, address, ia->address_length, 1);
       }));
-      /* *INDENT-ON* */
       ip6_mfib_interface_enable_disable (sw_if_index, 0);
+
+      if (0 != im6->fib_index_by_sw_if_index[sw_if_index])
+       fib_table_bind (FIB_PROTOCOL_IP6, sw_if_index, 0);
+      if (0 != im6->mfib_index_by_sw_if_index[sw_if_index])
+       mfib_table_bind (FIB_PROTOCOL_IP6, sw_if_index, 0);
+
+      /* Erase the lookup tables just in case */
+      im6->fib_index_by_sw_if_index[sw_if_index] = ~0;
+      im6->mfib_index_by_sw_if_index[sw_if_index] = ~0;
     }
 
   vnet_feature_enable_disable ("ip6-unicast", "ip6-not-enabled", sw_if_index,
@@ -738,7 +734,6 @@ VLIB_NODE_FN (ip6_lookup_node) (vlib_main_t * vm,
 
 static u8 *format_ip6_lookup_trace (u8 * s, va_list * args);
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_lookup_node) =
 {
   .name = "ip6-lookup",
@@ -747,7 +742,6 @@ VLIB_REGISTER_NODE (ip6_lookup_node) =
   .n_next_nodes = IP6_LOOKUP_N_NEXT,
   .next_nodes = IP6_LOOKUP_NEXT_NODES,
 };
-/* *INDENT-ON* */
 
 VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
                                      vlib_node_runtime_t * node,
@@ -925,7 +919,6 @@ VLIB_NODE_FN (ip6_load_balance_node) (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_load_balance_node) =
 {
   .name = "ip6-load-balance",
@@ -933,7 +926,6 @@ VLIB_REGISTER_NODE (ip6_load_balance_node) =
   .sibling_of = "ip6-lookup",
   .format_trace = format_ip6_lookup_trace,
 };
-/* *INDENT-ON* */
 
 typedef struct
 {
@@ -1217,23 +1209,17 @@ always_inline u8
 ip6_next_proto_is_tcp_udp (vlib_buffer_t * p0, ip6_header_t * ip0,
                           u32 * udp_offset0)
 {
-  u32 proto0;
-  proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_UDP, udp_offset0);
-  if (proto0 != IP_PROTOCOL_UDP)
-    {
-      proto0 = ip6_locate_header (p0, ip0, IP_PROTOCOL_TCP, udp_offset0);
-      proto0 = (proto0 == IP_PROTOCOL_TCP) ? proto0 : 0;
-    }
-  return proto0;
+  int nh = ip6_locate_header (p0, ip0, -1, udp_offset0);
+  if (nh > 0)
+    if (nh == IP_PROTOCOL_UDP || nh == IP_PROTOCOL_TCP)
+      return nh;
+  return 0;
 }
 
-/* *INDENT-OFF* */
-VNET_FEATURE_ARC_INIT (ip6_local) =
-{
-  .arc_name  = "ip6-local",
-  .start_nodes = VNET_FEATURES ("ip6-local"),
+VNET_FEATURE_ARC_INIT (ip6_local) = {
+  .arc_name = "ip6-local",
+  .start_nodes = VNET_FEATURES ("ip6-local", "ip6-receive"),
 };
-/* *INDENT-ON* */
 
 static_always_inline u8
 ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
@@ -1270,7 +1256,7 @@ ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
     }
 
   n_bytes_left -= n_this_buffer;
-  n_bytes_left -= p0->total_length_not_including_first_buffer;
+  n_bytes_left -= vlib_buffer_length_in_chain (vm, p0) - p0->current_length;
 
   if (n_bytes_left == 0)
     return 0;
@@ -1278,10 +1264,10 @@ ip6_tcp_udp_icmp_bad_length (vlib_main_t * vm, vlib_buffer_t * p0)
     return 1;
 }
 
-
 always_inline uword
-ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
-                 vlib_frame_t * frame, int head_of_feature_arc)
+ip6_local_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+                 vlib_frame_t *frame, int head_of_feature_arc,
+                 int is_receive_dpo)
 {
   ip6_main_t *im = &ip6_main;
   ip_lookup_main_t *lm = &im->lookup_main;
@@ -1313,7 +1299,7 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          vlib_prefetch_buffer_data (b[3], LOAD);
        }
 
-      ip6_error_t error[2];
+      vl_counter_ip6_enum_t error[2];
       error[0] = IP6_ERROR_UNKNOWN_PROTOCOL;
       error[1] = IP6_ERROR_UNKNOWN_PROTOCOL;
 
@@ -1334,15 +1320,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          flags[0] = b[0]->flags;
          flags[1] = b[1]->flags;
 
+         vnet_buffer_oflags_t oflags[2];
+         oflags[0] = vnet_buffer (b[0])->oflags;
+         oflags[1] = vnet_buffer (b[1])->oflags;
+
+         u32 l4_offload[2];
+         l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
+                         (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+                                       VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+         l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
+                         (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+                                       VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
          u32 good_l4_csum[2];
          good_l4_csum[0] =
-           flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-                       VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-                       VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+           (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
          good_l4_csum[1] =
-           flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-                       VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-                       VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+           (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
 
          u32 udp_offset[2] = { };
          u8 is_tcp_udp[2];
@@ -1461,6 +1455,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
            vnet_buffer (b[1])->sw_if_index[VLIB_TX] != ~0 ?
            vnet_buffer (b[1])->sw_if_index[VLIB_TX] :
            vnet_buffer (b[1])->ip.fib_index;
+
+         vnet_buffer (b[0])->ip.rx_sw_if_index =
+           vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+         vnet_buffer (b[1])->ip.rx_sw_if_index =
+           vnet_buffer (b[1])->sw_if_index[VLIB_RX];
+         if (is_receive_dpo)
+           {
+             const receive_dpo_t *rd0, *rd1;
+             rd0 =
+               receive_dpo_get (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
+             rd1 =
+               receive_dpo_get (vnet_buffer (b[1])->ip.adj_index[VLIB_TX]);
+             if (rd0->rd_sw_if_index != ~0)
+               vnet_buffer (b[0])->ip.rx_sw_if_index = rd0->rd_sw_if_index;
+             if (rd1->rd_sw_if_index != ~0)
+               vnet_buffer (b[1])->ip.rx_sw_if_index = rd1->rd_sw_if_index;
+           }
        }                       /* head_of_feature_arc */
 
       next[0] = lm->local_next_by_ip_protocol[ip[0]->protocol];
@@ -1482,16 +1493,16 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
            {
              u32 next32 = next[0];
              vnet_feature_arc_start (arc_index,
-                                     vnet_buffer (b[0])->sw_if_index
-                                     [VLIB_RX], &next32, b[0]);
+                                     vnet_buffer (b[0])->ip.rx_sw_if_index,
+                                     &next32, b[0]);
              next[0] = next32;
            }
          if (PREDICT_TRUE (ip6_unknown[1]))
            {
              u32 next32 = next[1];
              vnet_feature_arc_start (arc_index,
-                                     vnet_buffer (b[1])->sw_if_index
-                                     [VLIB_RX], &next32, b[1]);
+                                     vnet_buffer (b[1])->ip.rx_sw_if_index,
+                                     &next32, b[1]);
              next[1] = next32;
            }
        }
@@ -1516,11 +1527,15 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
 
          u32 flags = b[0]->flags;
-         u32 good_l4_csum =
-           flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-                    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-                    VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
 
+         vnet_buffer_oflags_t oflags = vnet_buffer (b[0])->oflags;
+
+         u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
+                          (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+                                     VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
+         u32 good_l4_csum =
+           (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
          u32 udp_offset;
          i16 len_diff = 0;
          u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
@@ -1540,8 +1555,9 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
          len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
 
-         u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
-           && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
+         u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
+                        !good_l4_csum &&
+                        !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
          if (PREDICT_FALSE (need_csum))
            {
              flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
@@ -1557,7 +1573,6 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 
          error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
-
          STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
                         IP6_ERROR_UDP_CHECKSUM,
                         "Wrong IP6 errors constants");
@@ -1584,6 +1599,16 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
            vnet_buffer (b[0])->sw_if_index[VLIB_TX] != ~0 ?
            vnet_buffer (b[0])->sw_if_index[VLIB_TX] :
            vnet_buffer (b[0])->ip.fib_index;
+
+         vnet_buffer (b[0])->ip.rx_sw_if_index =
+           vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+         if (is_receive_dpo)
+           {
+             receive_dpo_t *rd;
+             rd = receive_dpo_get (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]);
+             if (rd->rd_sw_if_index != ~0)
+               vnet_buffer (b[0])->ip.rx_sw_if_index = rd->rd_sw_if_index;
+           }
        }                       /* head_of_feature_arc */
 
       next[0] = lm->local_next_by_ip_protocol[ip->protocol];
@@ -1598,8 +1623,8 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
            {
              u32 next32 = next[0];
              vnet_feature_arc_start (arc_index,
-                                     vnet_buffer (b[0])->sw_if_index
-                                     [VLIB_RX], &next32, b[0]);
+                                     vnet_buffer (b[0])->ip.rx_sw_if_index,
+                                     &next32, b[0]);
              next[0] = next32;
            }
        }
@@ -1617,15 +1642,17 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 VLIB_NODE_FN (ip6_local_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
                               vlib_frame_t * frame)
 {
-  return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */ );
+  return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */,
+                          0 /* ip6_local_inline */);
 }
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_local_node) =
 {
   .name = "ip6-local",
   .vector_size = sizeof (u32),
   .format_trace = format_ip6_forward_next_trace,
+  .n_errors = IP6_N_ERROR,
+  .error_counters = ip6_error_counters,
   .n_next_nodes = IP_LOCAL_N_NEXT,
   .next_nodes =
   {
@@ -1633,19 +1660,32 @@ VLIB_REGISTER_NODE (ip6_local_node) =
     [IP_LOCAL_NEXT_PUNT] = "ip6-punt",
     [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip6-udp-lookup",
     [IP_LOCAL_NEXT_ICMP] = "ip6-icmp-input",
-    [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-full-reassembly",
+    [IP_LOCAL_NEXT_REASSEMBLY] = "ip6-local-full-reassembly",
   },
 };
-/* *INDENT-ON* */
+
+VLIB_NODE_FN (ip6_receive_local_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
+{
+  return ip6_local_inline (vm, node, frame, 1 /* head of feature arc */,
+                          1 /* is_receive_dpo */);
+}
+
+VLIB_REGISTER_NODE (ip6_receive_local_node) = {
+  .name = "ip6-receive",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip6_forward_next_trace,
+  .sibling_of = "ip6-local"
+};
 
 VLIB_NODE_FN (ip6_local_end_of_arc_node) (vlib_main_t * vm,
                                          vlib_node_runtime_t * node,
                                          vlib_frame_t * frame)
 {
-  return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */ );
+  return ip6_local_inline (vm, node, frame, 0 /* head of feature arc */,
+                          0 /* ip6_local_inline */);
 }
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_local_end_of_arc_node) = {
   .name = "ip6-local-end-of-arc",
   .vector_size = sizeof (u32),
@@ -1659,7 +1699,6 @@ VNET_FEATURE_INIT (ip6_local_end_of_arc, static) = {
   .node_name = "ip6-local-end-of-arc",
   .runs_before = 0, /* not before any other features */
 };
-/* *INDENT-ON* */
 
 #ifdef CLIB_MARCH_VARIANT
 extern vlib_node_registration_t ip6_local_node;
@@ -1768,8 +1807,8 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
            vlib_prefetch_buffer_header (p2, LOAD);
            vlib_prefetch_buffer_header (p3, LOAD);
 
-           CLIB_PREFETCH (p2->pre_data, 32, STORE);
-           CLIB_PREFETCH (p3->pre_data, 32, STORE);
+           clib_prefetch_store (p2->pre_data);
+           clib_prefetch_store (p3->pre_data);
 
            CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
            CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
@@ -1883,8 +1922,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
          if (p1->flags & VNET_BUFFER_F_GSO)
            ip1_len = gso_mtu_sz (p1);
 
-
-
          ip6_mtu_check (p0, ip0_len,
                         adj0[0].rewrite_header.max_l3_packet_bytes,
                         is_locally_originated0, &next0, is_midchain,
@@ -1893,18 +1930,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
                         adj1[0].rewrite_header.max_l3_packet_bytes,
                         is_locally_originated1, &next1, is_midchain,
                         &error1);
-
          /* Don't adjust the buffer for hop count issue; icmp-error node
           * wants to see the IP header */
          if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
            {
              p0->current_data -= rw_len0;
              p0->current_length += rw_len0;
-
              tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
              vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
              next0 = adj0[0].rewrite_header.next_index;
-
              if (PREDICT_FALSE
                  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
                vnet_feature_arc_start_w_cfg_index
@@ -1937,13 +1971,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
 
          if (is_midchain)
            {
-             /* before we paint on the next header, update the L4
-              * checksums if required, since there's no offload on a tunnel */
-             vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
-                                         1 /* is_ip6 */ );
-             vnet_calc_checksums_inline (vm, p1, 0 /* is_ip4 */ ,
-                                         1 /* is_ip6 */ );
-
              /* Guess we are only writing on ipv6 header. */
              vnet_rewrite_two_headers (adj0[0], adj1[0],
                                        ip0, ip1, sizeof (ip6_header_t));
@@ -2037,9 +2064,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
 
          if (is_midchain)
            {
-             vnet_calc_checksums_inline (vm, p0, 0 /* is_ip4 */ ,
-                                         1 /* is_ip6 */ );
-
              /* Guess we are only writing on ip6 header. */
              vnet_rewrite_one_header (adj0[0], ip0, sizeof (ip6_header_t));
            }
@@ -2071,7 +2095,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
                         adj0[0].rewrite_header.max_l3_packet_bytes,
                         is_locally_originated0, &next0, is_midchain,
                         &error0);
-
          /* Don't adjust the buffer for hop count issue; icmp-error node
           * wants to see the IP header */
          if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
@@ -2190,14 +2213,12 @@ VLIB_NODE_FN (ip6_mcast_midchain_node) (vlib_main_t * vm,
     return ip6_rewrite_inline (vm, node, frame, 0, 1, 1);
 }
 
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (ip6_midchain_node) =
-{
+VLIB_REGISTER_NODE (ip6_midchain_node) = {
   .name = "ip6-midchain",
   .vector_size = sizeof (u32),
   .format_trace = format_ip6_forward_next_trace,
   .sibling_of = "ip6-rewrite",
-  };
+};
 
 VLIB_REGISTER_NODE (ip6_rewrite_node) =
 {
@@ -2238,7 +2259,6 @@ VLIB_REGISTER_NODE (ip6_mcast_midchain_node) =
   .sibling_of = "ip6-rewrite",
 };
 
-/* *INDENT-ON* */
 
 /*
  * Hop-by-Hop handling
@@ -2252,7 +2272,6 @@ _(PROCESSED, "pkts with ip6 hop-by-hop options") \
 _(FORMAT, "incorrectly formatted hop-by-hop options") \
 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
 
-/* *INDENT-OFF* */
 typedef enum
 {
 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
@@ -2260,7 +2279,6 @@ typedef enum
 #undef _
   IP6_HOP_BY_HOP_N_ERROR,
 } ip6_hop_by_hop_error_t;
-/* *INDENT-ON* */
 
 /*
  * Primary h-b-h handler trace support
@@ -2290,7 +2308,6 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
   ip6_hop_by_hop_option_t *opt0, *limit0;
   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
   u8 type0;
-
   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
              hbh0->protocol, (hbh0->length + 1) << 3, total_len);
 
@@ -2313,9 +2330,8 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
            }
          else
            {
-             s =
-               format (s, "\n    unrecognized option %d length %d", type0,
-                       opt0->length);
+             s = format (s, "\n    unrecognized option %d length %d", type0,
+                         opt0->length);
            }
          opt0 =
            (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -2363,9 +2379,8 @@ format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
            }
          else
            {
-             s =
-               format (s, "\n    unrecognized option %d length %d", type0,
-                       opt0->length);
+             s = format (s, "\n    unrecognized option %d length %d", type0,
+                         opt0->length);
            }
          opt0 =
            (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -2690,7 +2705,6 @@ VLIB_NODE_FN (ip6_hop_by_hop_node) (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
 {
   .name = "ip6-hop-by-hop",
@@ -2702,7 +2716,6 @@ VLIB_REGISTER_NODE (ip6_hop_by_hop_node) =
   .error_strings = ip6_hop_by_hop_error_strings,
   .n_next_nodes = 0,
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 ip6_hop_by_hop_init (vlib_main_t * vm)
@@ -2954,14 +2967,12 @@ set_ip6_flow_hash_command_fn (vlib_main_t * vm,
  * @cliexend
  * @endparblock
 ?*/
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (set_ip6_flow_hash_command, static) = {
   .path = "set ip6 flow-hash",
   .short_help = "set ip6 flow-hash table <table-id> [src] [dst] [sport] "
                "[dport] [proto] [reverse] [flowlabel]",
   .function = set_ip6_flow_hash_command_fn,
 };
-/* *INDENT-ON* */
 
 static clib_error_t *
 show_ip6_local_command_fn (vlib_main_t * vm,
@@ -3002,19 +3013,16 @@ show_ip6_local_command_fn (vlib_main_t * vm,
  * 115
  * @cliexend
 ?*/
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (show_ip6_local, static) =
 {
   .path = "show ip6 local",
   .function = show_ip6_local_command_fn,
   .short_help = "show ip6 local",
 };
-/* *INDENT-ON* */
 
 #ifndef CLIB_MARCH_VARIANT
 int
-vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
-                            u32 table_index)
+vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
 {
   vnet_main_t *vnm = vnet_get_main ();
   vnet_interface_main_t *im = &vnm->interface_main;
@@ -3045,17 +3053,13 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
 
       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
                                                       sw_if_index);
-
-
       if (table_index != (u32) ~ 0)
        {
          dpo_id_t dpo = DPO_INVALID;
-
          dpo_set (&dpo,
                   DPO_CLASSIFY,
                   DPO_PROTO_IP6,
                   classify_dpo_create (DPO_PROTO_IP6, table_index));
-
          fib_table_entry_special_dpo_add (fib_index,
                                           &pfx,
                                           FIB_SOURCE_CLASSIFY,
@@ -3126,7 +3130,6 @@ set_ip6_classify_command_fn (vlib_main_t * vm,
  * Example of how to assign a classification table to an interface:
  * @cliexcmd{set ip6 classify intfc GigabitEthernet2/0/0 table-index 1}
 ?*/
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
 {
   .path = "set ip6 classify",
@@ -3134,7 +3137,6 @@ VLIB_CLI_COMMAND (set_ip6_classify_command, static) =
   "set ip6 classify intfc <interface> table-index <classify-idx>",
   .function = set_ip6_classify_command_fn,
 };
-/* *INDENT-ON* */
 
 /*
  * fd.io coding-style-patch-verification: ON