ipsec: compress ipsec_sa_t so data used by dataplane code fits in cacheline

[vpp.git] / src / plugins / dpdk / ipsec / esp_encrypt.c

diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c
index fb3f2a4..25815d9 100644 (file)
--- a/src/plugins/dpdk/ipsec/esp_encrypt.c
+++ b/src/plugins/dpdk/ipsec/esp_encrypt.c
@@ -22,6 +22,7 @@
 #include <vnet/ipsec/ipsec.h>
 #include <vnet/ipsec/esp.h>
 #include <vnet/udp/udp.h>
+#include <dpdk/buffer.h>
 #include <dpdk/ipsec/ipsec.h>
 #include <dpdk/device/dpdk.h>
 #include <dpdk/device/dpdk_priv.h>
@@ -63,8 +64,8 @@ static char *esp_encrypt_error_strings[] = {
 #undef _
 };
 
-vlib_node_registration_t dpdk_esp4_encrypt_node;
-vlib_node_registration_t dpdk_esp6_encrypt_node;
+extern vlib_node_registration_t dpdk_esp4_encrypt_node;
+extern vlib_node_registration_t dpdk_esp6_encrypt_node;
 
 typedef struct
 {
@@ -111,7 +112,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                         vlib_node_runtime_t * node,
                         vlib_frame_t * from_frame, int is_ip6)
 {
-  u32 n_left_from, *from, *to_next, next_index;
+  u32 n_left_from, *from, *to_next, next_index, thread_index;
   ipsec_main_t *im = &ipsec_main;
   u32 thread_idx = vlib_get_thread_index ();
   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
@@ -128,6 +129,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
 
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
+  thread_index = vm->thread_index;
 
   ret = crypto_alloc_ops (numa, ops, n_left_from);
   if (ret)
@@ -191,6 +193,8 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
          ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED);
 
          dpdk_op_priv_t *priv = crypto_op_get_priv (op);
+         /* store bi in op private */
+         priv->bi = bi0;
 
          u16 op_len =
            sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]);
@@ -277,7 +281,9 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
          orig_sz = b0->current_length;
 
          /* TODO multi-seg support - total_length_not_including_first_buffer */
-         sa0->total_data_size += b0->current_length;
+         vlib_increment_combined_counter
+           (&ipsec_sa_counters, thread_index, sa_index0,
+            1, b0->current_length);
 
          res->ops[res->n_ops] = op;
          res->bi[res->n_ops] = bi0;
@@ -291,13 +297,13 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
          trunc_size = auth_alg->trunc_size;
 
          /* if UDP encapsulation is used adjust the address of the IP header */
-         if (sa0->udp_encap && !is_ip6)
+         if (ipsec_sa_is_set_UDP_ENCAP (sa0) && !is_ip6)
            udp_encap_adv = sizeof (udp_header_t);
 
-         if (sa0->is_tunnel)
+         if (ipsec_sa_is_set_IS_TUNNEL (sa0))
            {
              rewrite_len = 0;
-             if (!is_ip6 && !sa0->is_tunnel_ip6)       /* ip4inip4 */
+             if (!is_ip6 && !ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))       /* ip4inip4 */
                {
                  /* in tunnel mode send it back to FIB */
                  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP4_LOOKUP;
@@ -327,7 +333,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                  oh0->ip4.dst_address.as_u32 =
                    sa0->tunnel_dst_addr.ip4.as_u32;
 
-                 if (sa0->udp_encap)
+                 if (ipsec_sa_is_set_UDP_ENCAP (sa0))
                    {
                      oh0->ip4.protocol = IP_PROTOCOL_UDP;
                      esp0 = &ouh0->esp;
@@ -337,8 +343,9 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                  esp0->spi = clib_host_to_net_u32 (sa0->spi);
                  esp0->seq = clib_host_to_net_u32 (sa0->seq);
                }
-             else if (is_ip6 && sa0->is_tunnel_ip6)    /* ip6inip6 */
+             else if (is_ip6 && ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))
                {
+                 /* ip6inip6 */
                  /* in tunnel mode send it back to FIB */
                  priv->next = DPDK_CRYPTO_INPUT_NEXT_IP6_LOOKUP;
 
@@ -412,7 +419,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                  memmove (dst, src, rewrite_len + ip_size);
                  oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP;
                  esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size);
-                 if (sa0->udp_encap)
+                 if (ipsec_sa_is_set_UDP_ENCAP (sa0))
                    {
                      oh0->ip4.protocol = IP_PROTOCOL_UDP;
                      esp0 = (esp_header_t *)
@@ -428,7 +435,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
              esp0->seq = clib_host_to_net_u32 (sa0->seq);
            }
 
-         if (sa0->udp_encap && ouh0)
+         if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
            {
              ouh0->udp.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
              ouh0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec);
@@ -444,7 +451,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
 
          /* The extra pad bytes would be overwritten by the digest */
          if (pad_bytes)
-           clib_memcpy (padding, pad_data, 16);
+           clib_memcpy_fast (padding, pad_data, 16);
 
          f0 = (esp_footer_t *) (padding + pad_bytes);
          f0->pad_length = pad_bytes;
@@ -461,7 +468,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
              oh0->ip4.length =
                clib_host_to_net_u16 (b0->current_length - rewrite_len);
              oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4);
-             if (sa0->udp_encap && ouh0)
+             if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0)
                {
                  ouh0->udp.length =
                    clib_host_to_net_u16 (clib_net_to_host_u16
@@ -470,8 +477,6 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                }
            }
 
-         vnet_buffer (b0)->sw_if_index[VLIB_RX] =
-           vnet_buffer (b0)->sw_if_index[VLIB_RX];
          b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 
          /* mbuf packet starts at ESP header */
@@ -508,7 +513,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
              aad[1] = clib_host_to_net_u32 (sa0->seq);
 
              /* aad[3] should always be 0 */
-             if (PREDICT_FALSE (sa0->use_esn))
+             if (PREDICT_FALSE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0)))
                aad[2] = clib_host_to_net_u32 (sa0->seq_hi);
              else
                aad[2] = 0;
@@ -517,7 +522,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
            {
              auth_len =
                vlib_buffer_get_tail (b0) - ((u8 *) esp0) - trunc_size;
-             if (sa0->use_esn)
+             if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))
                {
                  u32 *_digest = (u32 *) digest;
                  _digest[0] = clib_host_to_net_u32 (sa0->seq_hi);
@@ -536,9 +541,9 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
              tr->crypto_alg = sa0->crypto_alg;
              tr->integ_alg = sa0->integ_alg;
              u8 *p = vlib_buffer_get_current (b0);
-             if (!sa0->is_tunnel)
+             if (!ipsec_sa_is_set_IS_TUNNEL (sa0))
                p += vnet_buffer (b0)->ip.save_rewrite_length;
-             clib_memcpy (tr->packet_data, p, sizeof (tr->packet_data));
+             clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data));
            }
        }
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
@@ -549,7 +554,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                                   ESP_ENCRYPT_ERROR_RX_PKTS,
                                   from_frame->n_vectors);
 
-      crypto_enqueue_ops (vm, cwm, 1, dpdk_esp6_encrypt_node.index,
+      crypto_enqueue_ops (vm, cwm, dpdk_esp6_encrypt_node.index,
                          ESP_ENCRYPT_ERROR_ENQ_FAIL, numa);
     }
   else
@@ -558,7 +563,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm,
                                   ESP_ENCRYPT_ERROR_RX_PKTS,
                                   from_frame->n_vectors);
 
-      crypto_enqueue_ops (vm, cwm, 1, dpdk_esp4_encrypt_node.index,
+      crypto_enqueue_ops (vm, cwm, dpdk_esp4_encrypt_node.index,
                          ESP_ENCRYPT_ERROR_ENQ_FAIL, numa);
     }