X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fipsec%2Fipsec.h;h=054fe9b0223f912b4af0e52e7ccc218e4131e56c;hb=d9818dd68c162079f3ddb5443a78d0d91d55d0fe;hp=d7940345bfce7f3629aa5bbb575a4d7b275b059f;hpb=63c7e14f2e62caa1246349cfe341a93176ec4a43;p=vpp.git

diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h
index d7940345bfc..054fe9b0223 100644
--- a/src/plugins/dpdk/ipsec/ipsec.h
+++ b/src/plugins/dpdk/ipsec/ipsec.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 Intel and/or its affiliates.
+ * Copyright (c) 2017 Intel and/or its affiliates.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at:
@@ -16,6 +16,8 @@
 #define __DPDK_IPSEC_H__
 
 #include <vnet/vnet.h>
+#include <vppinfra/cache.h>
+#include <vnet/ipsec/ipsec.h>
 
 #undef always_inline
 #include <rte_config.h>
@@ -28,6 +30,21 @@
 #define always_inline static inline __attribute__ ((__always_inline__))
 #endif
 
+#define foreach_dpdk_crypto_input_next		\
+  _(DROP, "error-drop")				\
+  _(IP4_LOOKUP, "ip4-lookup")                   \
+  _(IP6_LOOKUP, "ip6-lookup")                   \
+  _(INTERFACE_OUTPUT, "interface-output")	\
+  _(DECRYPT4_POST, "dpdk-esp4-decrypt-post")     \
+  _(DECRYPT6_POST, "dpdk-esp6-decrypt-post")
+
+typedef enum
+{
+#define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
+  foreach_dpdk_crypto_input_next
+#undef _
+    DPDK_CRYPTO_INPUT_N_NEXT,
+} dpdk_crypto_input_next_t;
 
 #define MAX_QP_PER_LCORE 16
 
@@ -40,182 +57,328 @@ typedef struct
 
 typedef struct
 {
-  dpdk_gcm_cnt_blk cb;
-  union
-  {
-    u8 aad[12];
-    u8 icv[64];
-  };
-} dpdk_cop_priv_t;
+  u32 next;
+  dpdk_gcm_cnt_blk cb __attribute__ ((aligned (16)));
+  u8 aad[16];
+  u8 icv[32];
+} dpdk_op_priv_t;
+
+typedef struct
+{
+  u16 *resource_idx;
+  struct rte_crypto_op **ops;
+  u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG];
+  u16 auth_resource_idx[IPSEC_INTEG_N_ALG];
+    CLIB_CACHE_LINE_ALIGN_MARK (pad);
+} crypto_worker_main_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
+
+typedef struct
+{
+  CLIB_ALIGN_MARK (pad, 8);	/* align up to 8 bytes for 32bit builds */
+  char *name;
+  enum rte_crypto_sym_xform_type type;
+  u32 alg;
+  u8 key_len;
+  u8 iv_len;
+  u8 trunc_size;
+  u8 boundary;
+  u8 disabled;
+  u8 resources;
+} crypto_alg_t;
+
+typedef struct
+{
+  u16 *free_resources;
+  u16 *used_resources;
+  u8 cipher_support[IPSEC_CRYPTO_N_ALG];
+  u8 auth_support[IPSEC_INTEG_N_ALG];
+  u8 drv_id;
+  u8 numa;
+  u16 id;
+  const char *name;
+  u32 max_qp;
+  u64 features;
+} crypto_dev_t;
 
 typedef struct
 {
-  u8 cipher_algo;
-  u8 auth_algo;
-  u8 is_outbound;
-} crypto_worker_qp_key_t;
+  const char *name;
+  u16 *devs;
+} crypto_drv_t;
 
 typedef struct
 {
-  u16 dev_id;
+  u16 thread_idx;
+  u8 remove;
+  u8 drv_id;
+  u8 dev_id;
+  u8 numa;
   u16 qp_id;
-  u16 is_outbound;
-  i16 inflights;
+  u16 inflights[2];
+  u16 n_ops;
+  u16 __unused;
+  struct rte_crypto_op *ops[VLIB_FRAME_SIZE];
   u32 bi[VLIB_FRAME_SIZE];
-  struct rte_crypto_op *cops[VLIB_FRAME_SIZE];
-  struct rte_crypto_op **free_cops;
-} crypto_qp_data_t;
+} crypto_resource_t __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES)));
+
+typedef struct
+{
+  u64 ts;
+  struct rte_cryptodev_sym_session *session;
+} crypto_session_disposal_t;
 
 typedef struct
 {
-  u8 qp_index;
-  void *sess;
-} crypto_sa_session_t;
+  CLIB_ALIGN_MARK (pad, 16);	/* align up to 16 bytes for 32bit builds */
+  struct rte_cryptodev_sym_session *session;
+  u64 dev_mask;
+} crypto_session_by_drv_t;
 
 typedef struct
 {
-  crypto_sa_session_t *sa_sess_d[2];
-  crypto_qp_data_t *qp_data;
-  uword *algo_qp_map;
-} crypto_worker_main_t;
+  /* Required for vec_validate_aligned */
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  struct rte_mempool *crypto_op;
+  struct rte_mempool *session_h;
+  struct rte_mempool **session_drv;
+  crypto_session_disposal_t *session_disposal;
+  uword *session_by_sa_index;
+  u64 crypto_op_get_failed;
+  u64 session_h_failed;
+  u64 *session_drv_failed;
+  crypto_session_by_drv_t *session_by_drv_id_and_sa_index;
+  clib_spinlock_t lockp;
+} crypto_data_t;
 
 typedef struct
 {
-  struct rte_mempool **cop_pools;
   crypto_worker_main_t *workers_main;
+  crypto_dev_t *dev;
+  crypto_resource_t *resource;
+  crypto_alg_t *cipher_algs;
+  crypto_alg_t *auth_algs;
+  crypto_data_t *data;
+  crypto_drv_t *drv;
+  u64 session_timeout;		/* nsec */
   u8 enabled;
 } dpdk_crypto_main_t;
 
-dpdk_crypto_main_t dpdk_crypto_main;
+extern dpdk_crypto_main_t dpdk_crypto_main;
+
+static const u8 pad_data[] =
+  { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 };
+
+void crypto_auto_placement (void);
+
+clib_error_t *create_sym_session (struct rte_cryptodev_sym_session **session,
+				  u32 sa_idx, crypto_resource_t * res,
+				  crypto_worker_main_t * cwm, u8 is_outbound);
+
+static_always_inline u32
+crypto_op_len (void)
+{
+  const u32 align = 4;
+  u32 op_size =
+    sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
+
+  return ((op_size + align - 1) & ~(align - 1)) + sizeof (dpdk_op_priv_t);
+}
 
-extern vlib_node_registration_t dpdk_crypto_input_node;
+static_always_inline u32
+crypto_op_get_priv_offset (void)
+{
+  const u32 align = 16;
+  u32 offset;
+
+  offset = sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
+  offset = (offset + align - 1) & ~(align - 1);
+
+  return offset;
+}
+
+static_always_inline dpdk_op_priv_t *
+crypto_op_get_priv (struct rte_crypto_op * op)
+{
+  return (dpdk_op_priv_t *) (((u8 *) op) + crypto_op_get_priv_offset ());
+}
 
-#define CRYPTO_N_FREE_COPS (VLIB_FRAME_SIZE * 3)
 
 static_always_inline void
-crypto_alloc_cops ()
+add_session_by_drv_and_sa_idx (struct rte_cryptodev_sym_session *session,
+			       crypto_data_t * data, u32 drv_id, u32 sa_idx)
+{
+  crypto_session_by_drv_t *sbd;
+  vec_validate_aligned (data->session_by_drv_id_and_sa_index, sa_idx,
+			CLIB_CACHE_LINE_BYTES);
+  sbd = vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
+  sbd->dev_mask |= 1L << drv_id;
+  sbd->session = session;
+}
+
+static_always_inline struct rte_cryptodev_sym_session *
+get_session_by_drv_and_sa_idx (crypto_data_t * data, u32 drv_id, u32 sa_idx)
+{
+  crypto_session_by_drv_t *sess_by_sa;
+  if (_vec_len (data->session_by_drv_id_and_sa_index) <= sa_idx)
+    return NULL;
+  sess_by_sa =
+    vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
+  return (sess_by_sa->dev_mask & (1L << drv_id)) ? sess_by_sa->session : NULL;
+}
+
+static_always_inline clib_error_t *
+crypto_get_session (struct rte_cryptodev_sym_session ** session,
+		    u32 sa_idx,
+		    crypto_resource_t * res,
+		    crypto_worker_main_t * cwm, u8 is_outbound)
 {
   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
-  u32 thread_index = vlib_get_thread_index ();
-  crypto_worker_main_t *cwm = &dcm->workers_main[thread_index];
-  unsigned socket_id = rte_socket_id ();
-  crypto_qp_data_t *qpd;
+  crypto_data_t *data;
+  struct rte_cryptodev_sym_session *sess;
 
-  /* *INDENT-OFF* */
-  vec_foreach (qpd, cwm->qp_data)
-    {
-      u32 l = vec_len (qpd->free_cops);
+  data = vec_elt_at_index (dcm->data, res->numa);
+  sess = get_session_by_drv_and_sa_idx (data, res->drv_id, sa_idx);
 
-      if (PREDICT_FALSE (l < VLIB_FRAME_SIZE))
-	{
-	  u32 n_alloc;
+  if (PREDICT_FALSE (!sess))
+    return create_sym_session (session, sa_idx, res, cwm, is_outbound);
 
-	  if (PREDICT_FALSE (!qpd->free_cops))
-	    vec_alloc (qpd->free_cops, CRYPTO_N_FREE_COPS);
+  session[0] = sess;
 
-	  n_alloc = rte_crypto_op_bulk_alloc (dcm->cop_pools[socket_id],
-					      RTE_CRYPTO_OP_TYPE_SYMMETRIC,
-					      &qpd->free_cops[l],
-					      CRYPTO_N_FREE_COPS - l - 1);
+  return NULL;
+}
 
-	  _vec_len (qpd->free_cops) = l + n_alloc;
-	}
-    }
+static_always_inline u16
+get_resource (crypto_worker_main_t * cwm, ipsec_sa_t * sa)
+{
+  u16 cipher_res = cwm->cipher_resource_idx[sa->crypto_alg];
+  u16 auth_res = cwm->auth_resource_idx[sa->integ_alg];
+  u8 is_aead;
+
+  /* Not allowed to setup SA with no-aead-cipher/NULL or NULL/NULL */
+
+  is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) ||
+	     (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) ||
+	     (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256));
+
+  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE)
+    return auth_res;
+
+  if (cipher_res == auth_res)
+    return cipher_res;
+
+  if (is_aead)
+    return cipher_res;
+
+  return (u16) ~ 0;
+}
+
+static_always_inline i32
+crypto_alloc_ops (u8 numa, struct rte_crypto_op ** ops, u32 n)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
+  i32 ret;
+
+  ret = rte_mempool_get_bulk (data->crypto_op, (void **) ops, n);
+
+  /* *INDENT-OFF* */
+  data->crypto_op_get_failed += ! !ret;
   /* *INDENT-ON* */
+
+  return ret;
 }
 
 static_always_inline void
-crypto_free_cop (crypto_qp_data_t * qpd, struct rte_crypto_op **cops, u32 n)
+crypto_free_ops (u8 numa, struct rte_crypto_op **ops, u32 n)
 {
-  u32 l = vec_len (qpd->free_cops);
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
+
+  if (!n)
+    return;
 
-  if (l + n >= CRYPTO_N_FREE_COPS)
+  rte_mempool_put_bulk (data->crypto_op, (void **) ops, n);
+}
+
+static_always_inline void
+crypto_enqueue_ops (vlib_main_t * vm, crypto_worker_main_t * cwm, u8 outbound,
+		    u32 node_index, u32 error, u8 numa)
+{
+  dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
+  crypto_resource_t *res;
+  u16 *res_idx;
+
+  /* *INDENT-OFF* */
+  vec_foreach (res_idx, cwm->resource_idx)
     {
-      l -= VLIB_FRAME_SIZE;
-      rte_mempool_put_bulk (cops[0]->mempool,
-			    (void **) &qpd->free_cops[l], VLIB_FRAME_SIZE);
+      u16 enq;
+      res = vec_elt_at_index (dcm->resource, res_idx[0]);
+
+      if (!res->n_ops)
+	continue;
+
+      enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id + outbound,
+					 res->ops, res->n_ops);
+      res->inflights[outbound] += enq;
+
+      if (PREDICT_FALSE (enq < res->n_ops))
+	{
+	  crypto_free_ops (numa, &res->ops[enq], res->n_ops - enq);
+	  vlib_buffer_free (vm, &res->bi[enq], res->n_ops - enq);
+
+          vlib_node_increment_counter (vm, node_index, error,
+				       res->n_ops - enq);
+        }
+      res->n_ops = 0;
     }
-  clib_memcpy (&qpd->free_cops[l], cops, sizeof (*cops) * n);
+  /* *INDENT-ON* */
+}
 
-  _vec_len (qpd->free_cops) = l + n;
+static_always_inline void
+crypto_set_icb (dpdk_gcm_cnt_blk * icb, u32 salt, u32 seq, u32 seq_hi)
+{
+  icb->salt = salt;
+  icb->iv[0] = seq;
+  icb->iv[1] = seq_hi;
 }
 
-static_always_inline int
-check_algo_is_supported (const struct rte_cryptodev_capabilities *cap,
-			 char *name)
+static_always_inline void
+crypto_op_setup (u8 is_aead, struct rte_mbuf *mb0,
+		 struct rte_crypto_op *op, void *session,
+		 u32 cipher_off, u32 cipher_len,
+		 u32 auth_off, u32 auth_len,
+		 u8 * aad, u8 * digest, u64 digest_paddr)
 {
-  struct
-  {
-    uint8_t cipher_algo;
-    enum rte_crypto_sym_xform_type type;
-    union
-    {
-      enum rte_crypto_auth_algorithm auth;
-      enum rte_crypto_cipher_algorithm cipher;
-    };
-    char *name;
-  } supported_algo[] =
-  {
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_NULL,.name = "NULL"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_CBC,.name = "AES_CBC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_CTR,.name = "AES_CTR"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_3DES_CBC,.name = "3DES-CBC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher =
-	RTE_CRYPTO_CIPHER_AES_GCM,.name = "AES-GCM"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA1_HMAC,.name = "HMAC-SHA1"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA256_HMAC,.name = "HMAC-SHA256"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA384_HMAC,.name = "HMAC-SHA384"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_SHA512_HMAC,.name = "HMAC-SHA512"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_AES_XCBC_MAC,.name = "AES-XCBC-MAC"},
-    {
-    .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth =
-	RTE_CRYPTO_AUTH_AES_GCM,.name = "AES-GCM"},
-    {
-      /* tail */
-  .type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED},};
-  uint32_t i = 0;
+  struct rte_crypto_sym_op *sym_op;
+
+  sym_op = (struct rte_crypto_sym_op *) (op + 1);
 
-  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
-    return -1;
+  sym_op->m_src = mb0;
+  sym_op->session = session;
 
-  while (supported_algo[i].type != RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED)
+  if (is_aead)
     {
-      if (cap->sym.xform_type == supported_algo[i].type)
-	{
-	  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
-	       cap->sym.cipher.algo == supported_algo[i].cipher) ||
-	      (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH &&
-	       cap->sym.auth.algo == supported_algo[i].auth))
-	    {
-	      if (name)
-		strcpy (name, supported_algo[i].name);
-	      return 0;
-	    }
-	}
-
-      i++;
+      sym_op->aead.data.offset = cipher_off;
+      sym_op->aead.data.length = cipher_len;
+
+      sym_op->aead.aad.data = aad;
+      sym_op->aead.aad.phys_addr =
+	op->phys_addr + (uintptr_t) aad - (uintptr_t) op;
+
+      sym_op->aead.digest.data = digest;
+      sym_op->aead.digest.phys_addr = digest_paddr;
     }
+  else
+    {
+      sym_op->cipher.data.offset = cipher_off;
+      sym_op->cipher.data.length = cipher_len;
 
-  return -1;
+      sym_op->auth.data.offset = auth_off;
+      sym_op->auth.data.length = auth_len;
+
+      sym_op->auth.digest.data = digest;
+      sym_op->auth.digest.phys_addr = digest_paddr;
+    }
 }
 
 #endif /* __DPDK_IPSEC_H__ */