ipsec: re-enable DPDK IPSec for tunnel decap/encap (VPP-1823)

[vpp.git] / src / plugins / dpdk / ipsec / ipsec.h

diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h
index dc11dfc..7416743 100644 (file)
--- a/src/plugins/dpdk/ipsec/ipsec.h
+++ b/src/plugins/dpdk/ipsec/ipsec.h
@@ -30,12 +30,17 @@
 #define always_inline static inline __attribute__ ((__always_inline__))
 #endif
 
+#define DPDK_CRYPTO_N_QUEUE_DESC  2048
+#define DPDK_CRYPTO_NB_SESS_OBJS  20000
+
 #define foreach_dpdk_crypto_input_next         \
   _(DROP, "error-drop")                                \
   _(IP4_LOOKUP, "ip4-lookup")                   \
   _(IP6_LOOKUP, "ip6-lookup")                   \
   _(INTERFACE_OUTPUT, "interface-output")      \
-  _(DECRYPT_POST, "dpdk-esp-decrypt-post")
+  _(MIDCHAIN, "adj-midchain-tx")                 \
+  _(DECRYPT4_POST, "dpdk-esp4-decrypt-post")     \
+  _(DECRYPT6_POST, "dpdk-esp6-decrypt-post")
 
 typedef enum
 {
@@ -57,9 +62,12 @@ typedef struct
 typedef struct
 {
   u32 next;
-  dpdk_gcm_cnt_blk cb __attribute__ ((aligned (16)));
+  u32 bi;
+  u8 encrypt;
+    CLIB_ALIGN_MARK (mark0, 16);
+  dpdk_gcm_cnt_blk cb;
   u8 aad[16];
-  u8 icv[32];
+  u8 icv[32];                  /* XXX last 16B in next cache line */
 } dpdk_op_priv_t;
 
 typedef struct
@@ -68,11 +76,12 @@ typedef struct
   struct rte_crypto_op **ops;
   u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG];
   u16 auth_resource_idx[IPSEC_INTEG_N_ALG];
-    CLIB_CACHE_LINE_ALIGN_MARK (pad);
-} crypto_worker_main_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
+    CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+} crypto_worker_main_t;
 
 typedef struct
 {
+  CLIB_ALIGN_MARK (pad, 8);    /* align up to 8 bytes for 32bit builds */
   char *name;
   enum rte_crypto_sym_xform_type type;
   u32 alg;
@@ -82,7 +91,7 @@ typedef struct
   u8 boundary;
   u8 disabled;
   u8 resources;
-} crypto_alg_t __attribute__ ((aligned (8)));
+} crypto_alg_t;
 
 typedef struct
 {
@@ -117,7 +126,8 @@ typedef struct
   u16 __unused;
   struct rte_crypto_op *ops[VLIB_FRAME_SIZE];
   u32 bi[VLIB_FRAME_SIZE];
-} crypto_resource_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
+    CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+} crypto_resource_t;
 
 typedef struct
 {
@@ -129,12 +139,11 @@ typedef struct
 {
   struct rte_cryptodev_sym_session *session;
   u64 dev_mask;
+    CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
 } crypto_session_by_drv_t;
 
 typedef struct
 {
-  /* Required for vec_validate_aligned */
-  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
   struct rte_mempool *crypto_op;
   struct rte_mempool *session_h;
   struct rte_mempool **session_drv;
@@ -145,6 +154,8 @@ typedef struct
   u64 *session_drv_failed;
   crypto_session_by_drv_t *session_by_drv_id_and_sa_index;
   clib_spinlock_t lockp;
+  /* Required for vec_validate_aligned */
+    CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 } crypto_data_t;
 
 typedef struct
@@ -298,8 +309,8 @@ crypto_free_ops (u8 numa, struct rte_crypto_op **ops, u32 n)
 }
 
 static_always_inline void
-crypto_enqueue_ops (vlib_main_t * vm, crypto_worker_main_t * cwm, u8 outbound,
-                   u32 node_index, u32 error, u8 numa)
+crypto_enqueue_ops (vlib_main_t * vm, crypto_worker_main_t * cwm,
+                   u32 node_index, u32 error, u8 numa, u8 encrypt)
 {
   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
   crypto_resource_t *res;
@@ -308,15 +319,18 @@ crypto_enqueue_ops (vlib_main_t * vm, crypto_worker_main_t * cwm, u8 outbound,
   /* *INDENT-OFF* */
   vec_foreach (res_idx, cwm->resource_idx)
     {
-      u16 enq;
+      u16 enq, n_ops;
       res = vec_elt_at_index (dcm->resource, res_idx[0]);
 
       if (!res->n_ops)
        continue;
 
-      enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id + outbound,
-                                        res->ops, res->n_ops);
-      res->inflights[outbound] += enq;
+      n_ops = (DPDK_CRYPTO_N_QUEUE_DESC / 2) - res->inflights[encrypt];
+      n_ops = res->n_ops < n_ops ? res->n_ops : n_ops;
+      enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id,
+                                        res->ops, n_ops);
+      ASSERT (n_ops == enq);
+      res->inflights[encrypt] += enq;
 
       if (PREDICT_FALSE (enq < res->n_ops))
        {