octeon: add direct mode changes in crypto datapath

This patch introduces support for direct mode crypto
submission on CPT. For multi-segmented buffers,
scatter-gather submission mode will be utilized.

Type: feature

Signed-off-by: Nithinsen Kaithakadan <[email protected]>
Change-Id: Idb99e6c9ea49028e11d3bf530c9559719c988252

diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c
index 1c3aa42..b612420 100644 (file)
--- a/src/plugins/dev_octeon/crypto.c
+++ b/src/plugins/dev_octeon/crypto.c
@@ -63,6 +63,9 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index,
   oct_crypto_sess_t *session;
   vnet_crypto_key_t *key;
   oct_crypto_key_t *ckey;
+  oct_crypto_dev_t *ocd;
+
+  ocd = ocm->crypto_dev[op_type];
 
   key = vnet_crypto_get_key (key_index);
 
@@ -89,6 +92,7 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index,
       session = oct_crypto_session_alloc (vm, op_type);
       if (session == NULL)
        return -1;
+      session->crypto_dev = ocd;
     }
 
   oct_map_keyindex_to_session (session, key_index, op_type);
@@ -117,6 +121,12 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
            ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index);
          ckey_linked->sess = NULL;
        }
+
+      /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+      if (oct_hw_ctx_cache_enable ())
+       roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf,
+                             &ckey->sess->cpt_ctx.se_ctx, true);
+
       oct_plt_init_param.oct_plt_free (ckey->sess);
       ckey->sess = NULL;
     }
@@ -134,6 +144,11 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
          ckey_linked->sess = NULL;
        }
 
+      /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+      if (oct_hw_ctx_cache_enable ())
+       roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf,
+                             &ckey->sess->cpt_ctx.se_ctx, true);
+
       oct_plt_init_param.oct_plt_free (ckey->sess);
       ckey->sess = NULL;
     }
@@ -1060,12 +1075,11 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens,
 }
 
 static_always_inline int
-oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst,
-                          const bool is_aead, u8 aad_length, u8 *payload,
-                          vnet_crypto_async_frame_elt_t *elts, void *mdata,
-                          u32 cipher_data_length, u32 cipher_data_offset,
-                          u32 auth_data_length, u32 auth_data_offset,
-                          vlib_buffer_t *b, u16 adj_len)
+oct_crypto_scatter_gather_mode (
+  oct_crypto_sess_t *sess, struct cpt_inst_s *inst, const bool is_aead,
+  u8 aad_length, u8 *payload, vnet_crypto_async_frame_elt_t *elts, void *mdata,
+  u32 cipher_data_length, u32 cipher_data_offset, u32 auth_data_length,
+  u32 auth_data_offset, vlib_buffer_t *b, u16 adj_len)
 {
   struct roc_se_fc_params fc_params = { 0 };
   struct roc_se_ctx *ctx = &sess->cpt_ctx;
@@ -1163,6 +1177,9 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt)
   inst_w7.u64 = 0;
   inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx;
 
+  if (oct_hw_ctx_cache_enable ())
+    inst_w7.s.ctx_val = 1;
+
   /* Set the engine group */
   inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
 
@@ -1299,6 +1316,13 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
       return -1;
     }
 
+  sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT;
+  else
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT;
+
   return 0;
 }
 
@@ -1352,6 +1376,13 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
       return -1;
     }
 
+  sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT;
+  else
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT;
+
   if (enc_type == ROC_SE_CHACHA20)
     sess->cpt_ctx.template_w4.s.opcode_minor |= BIT (5);
 
@@ -1387,6 +1418,9 @@ oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session,
   session->cpt_inst_w7 =
     oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt);
 
+  if (oct_hw_ctx_cache_enable ())
+    roc_se_ctx_init (&session->cpt_ctx);
+
   session->initialised = 1;
 
   return 0;
@@ -1405,6 +1439,138 @@ oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, u32 index,
     f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
 }
 
+static_always_inline void
+oct_crypto_direct_mode_linked (vlib_buffer_t *buffer, struct cpt_inst_s *inst,
+                              oct_crypto_sess_t *sess,
+                              oct_crypto_inflight_req_t *infl_req, u8 aad_len)
+{
+  u32 encr_offset, auth_offset, iv_offset;
+  vnet_crypto_async_frame_elt_t *elts;
+  union cpt_inst_w4 cpt_inst_w4;
+  u64 *offset_control_word;
+  u32 crypto_total_length;
+  u32 auth_dlen, enc_dlen;
+  u32 enc_auth_len;
+
+  elts = infl_req->fe;
+  enc_auth_len = elts->crypto_total_length + elts->integ_length_adj;
+  crypto_total_length = elts->crypto_total_length;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    {
+      /*
+       * Position the offset control word so that it does not
+       * overlap with the IV.
+       */
+      offset_control_word = (void *) (buffer->data) - ROC_SE_OFF_CTRL_LEN - 4;
+
+      iv_offset =
+       (void *) elts->iv - (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+    }
+  else
+    {
+      offset_control_word = (void *) (elts->iv) - ROC_SE_OFF_CTRL_LEN;
+      iv_offset = 0;
+    }
+
+  encr_offset = (void *) (buffer->data + elts->crypto_start_offset) -
+               (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  auth_offset = (void *) (buffer->data + elts->integ_start_offset) -
+               (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  *offset_control_word = clib_host_to_net_u64 (
+    ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset));
+
+  cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64;
+
+  cpt_inst_w4.s.param1 = crypto_total_length;
+  cpt_inst_w4.s.param2 = enc_auth_len;
+
+  auth_dlen = auth_offset + enc_auth_len + ROC_SE_OFF_CTRL_LEN;
+  enc_dlen = encr_offset + crypto_total_length + ROC_SE_OFF_CTRL_LEN;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    cpt_inst_w4.s.dlen = auth_dlen + sess->cpt_ctx.mac_len;
+  else
+    {
+      /*
+       * In the case of ESN, 4 bytes of the seqhi will be stored at the end of
+       * the cipher. This data must be overwritten by the digest data during
+       * the dequeue process.
+       */
+      if (auth_dlen > enc_dlen)
+       infl_req->esn_enabled = true;
+
+      cpt_inst_w4.s.dlen = auth_dlen;
+    }
+
+  infl_req->mac_len = sess->cpt_ctx.mac_len;
+
+  inst->dptr = (uint64_t) offset_control_word;
+  inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN);
+  inst->w4.u64 = cpt_inst_w4.u64;
+}
+
+static_always_inline void
+oct_crypto_direct_mode_aead (vlib_buffer_t *buffer, struct cpt_inst_s *inst,
+                            oct_crypto_sess_t *sess,
+                            oct_crypto_inflight_req_t *infl_req, u8 aad_len)
+{
+  u32 encr_offset, auth_offset, iv_offset;
+  u32 auth_copy_offset, iv_copy_offset;
+  vnet_crypto_async_frame_elt_t *elts;
+  union cpt_inst_w4 cpt_inst_w4;
+  u64 *offset_control_word;
+  u32 crypto_total_length;
+
+  elts = infl_req->fe;
+  crypto_total_length = elts->crypto_total_length;
+
+  ((u32 *) elts->iv)[3] = clib_host_to_net_u32 (0x1);
+
+  offset_control_word = (void *) (elts->aad) - ROC_SE_OFF_CTRL_LEN;
+  encr_offset = (void *) (buffer->data + elts->crypto_start_offset) -
+               (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  iv_offset = elts->iv - elts->aad;
+  auth_offset = encr_offset - aad_len;
+
+  *offset_control_word = clib_host_to_net_u64 (
+    ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset));
+
+  cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64;
+
+  cpt_inst_w4.s.param1 = crypto_total_length;
+  cpt_inst_w4.s.param2 = crypto_total_length + aad_len;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    cpt_inst_w4.s.dlen = encr_offset + elts->crypto_total_length +
+                        ROC_SE_OFF_CTRL_LEN + sess->cpt_ctx.mac_len;
+  else
+    cpt_inst_w4.s.dlen =
+      encr_offset + elts->crypto_total_length + ROC_SE_OFF_CTRL_LEN;
+
+  inst->dptr = (uint64_t) offset_control_word;
+  inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN);
+  inst->w4.u64 = cpt_inst_w4.u64;
+
+  /*
+   * CPT hardware requires the AAD to be followed by the cipher packet.
+   * Therefore, maintain a copy of the AAD and IV in the inflight request,
+   * and write the AAD in front of the cipher data before submission.
+   */
+  auth_copy_offset = encr_offset - sess->cpt_ctx.mac_len;
+  iv_copy_offset = encr_offset - 8;
+
+  clib_memcpy_fast (infl_req->aad,
+                   ((void *) inst->dptr) + auth_copy_offset + 8, 8);
+  clib_memcpy_fast (infl_req->iv, ((void *) inst->dptr) + iv_copy_offset + 8,
+                   8);
+  clib_memcpy_fast (((void *) inst->dptr) + encr_offset + ROC_SE_OFF_CTRL_LEN -
+                     aad_len,
+                   elts->aad, aad_len);
+
+  infl_req->aead_algo = true;
+}
+
 static_always_inline int
 oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
                            const u8 is_aead, u8 aad_len, const u8 type)
@@ -1480,54 +1646,72 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
 
       if (is_aead)
        {
-         dptr_start_ptr =
-           (u64) (buffer->data + (elts->crypto_start_offset - aad_iv));
-         curr_ptr = (u64) (buffer->data + buffer->current_data);
-         adj_len = (u16) (dptr_start_ptr - curr_ptr);
-
-         crypto_total_length = elts->crypto_total_length;
-         crypto_start_offset = aad_iv;
-         integ_start_offset = 0;
-
-         ret = oct_crypto_fill_fc_params (
-           sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
-           ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
-           crypto_total_length /* cipher_len */,
-           crypto_start_offset /* cipher_offset */, 0 /* auth_len */,
-           integ_start_offset /* auth_off */, buffer, adj_len);
-         if (PREDICT_FALSE (ret < 0))
+         if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT)
            {
-             oct_crypto_update_frame_error_status (
-               frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-             return -1;
+             dptr_start_ptr =
+               (u64) (buffer->data + (elts->crypto_start_offset - aad_iv));
+             curr_ptr = (u64) (buffer->data + buffer->current_data);
+             adj_len = (u16) (dptr_start_ptr - curr_ptr);
+
+             crypto_total_length = elts->crypto_total_length;
+             crypto_start_offset = aad_iv;
+             integ_start_offset = 0;
+
+             ret = oct_crypto_scatter_gather_mode (
+               sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
+               ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
+               crypto_total_length /* cipher_len */,
+               crypto_start_offset /* cipher_offset */, 0 /* auth_len */,
+               integ_start_offset /* auth_off */, buffer, adj_len);
+
+             if (PREDICT_FALSE (ret < 0))
+               {
+                 oct_crypto_update_frame_error_status (
+                   frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+                 return -1;
+               }
+           }
+         else
+           {
+             oct_crypto_direct_mode_aead (buffer, inst + i, sess, infl_req,
+                                          aad_len);
            }
        }
       else
        {
-         dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset);
-
-         enc_auth_len = elts->crypto_total_length + elts->integ_length_adj;
-
-         curr_ptr = (u64) (buffer->data + buffer->current_data);
-         adj_len = (u16) (dptr_start_ptr - curr_ptr);
-
-         crypto_total_length = elts->crypto_total_length;
-         crypto_start_offset =
-           elts->crypto_start_offset - elts->integ_start_offset;
-         integ_start_offset = 0;
-
-         ret = oct_crypto_fill_fc_params (
-           sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
-           ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
-           crypto_total_length /* cipher_len */,
-           crypto_start_offset /* cipher_offset */,
-           enc_auth_len /* auth_len */, integ_start_offset /* auth_off */,
-           buffer, adj_len);
-         if (PREDICT_FALSE (ret < 0))
+         if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT)
            {
-             oct_crypto_update_frame_error_status (
-               frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-             return -1;
+             dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset);
+
+             curr_ptr = (u64) (buffer->data + buffer->current_data);
+             adj_len = (u16) (dptr_start_ptr - curr_ptr);
+
+             crypto_start_offset =
+               elts->crypto_start_offset - elts->integ_start_offset;
+             integ_start_offset = 0;
+             enc_auth_len =
+               elts->crypto_total_length + elts->integ_length_adj;
+             crypto_total_length = elts->crypto_total_length;
+
+             ret = oct_crypto_scatter_gather_mode (
+               sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
+               ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
+               crypto_total_length /* cipher_len */,
+               crypto_start_offset /* cipher_offset */,
+               enc_auth_len /* auth_len */, integ_start_offset /* auth_off */,
+               buffer, adj_len);
+
+             if (PREDICT_FALSE (ret < 0))
+               {
+                 oct_crypto_update_frame_error_status (
+                   frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+                 return -1;
+               }
+           }
+         else
+           {
+             oct_crypto_direct_mode_linked (buffer, inst + i, sess, infl_req,
+                                            aad_len);
            }
        }
 
@@ -1632,6 +1816,7 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
   vnet_crypto_async_frame_t *frame;
   volatile union cpt_res_s *res;
   bool last_elts_processed;
+  vlib_buffer_t *buffer;
 
   pend_q = &ocm->pend_q[vlib_get_thread_index ()];
 
@@ -1659,6 +1844,24 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
            status = fe->status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR;
        }
 
+      buffer =
+       vlib_get_buffer (vm, infl_req->frame->buffer_indices[infl_req->index]);
+
+      /*
+       * For AEAD, copy the AAD and IV back to their original positions.
+       * If ESN is enabled (in case of linked algo), overwrite the ESN
+       * seqhi at the end of the cipher with the digest data.
+       */
+      if (infl_req->aead_algo)
+       {
+         clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 8,
+                           infl_req->iv, 8);
+         clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 16,
+                           infl_req->aad, 8);
+       }
+      else if (infl_req->esn_enabled)
+       clib_memcpy_fast (fe->digest, fe->digest + 4, infl_req->mac_len);
+
       clib_memset ((void *) &infl_req->res, 0, sizeof (union cpt_res_s));
       last_elts_processed = infl_req->last_elts;
       OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc);

diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h
index d7a501e..b003a60 100644 (file)
--- a/src/plugins/dev_octeon/crypto.h
+++ b/src/plugins/dev_octeon/crypto.h
@@ -120,6 +120,17 @@ typedef struct
   vnet_crypto_async_frame_t *frame;
   /** Async frame element */
   vnet_crypto_async_frame_elt_t *fe;
+  /** AAD meta data */
+  u8 aad[8];
+  /** IV meta data */
+  u8 iv[16];
+  /** Digest len */
+  u8 mac_len;
+  /** aead */
+  bool aead_algo;
+  /** Set when encrypting linked algo with esn.
+   * To move digest data */
+  bool esn_enabled;
   /** Set if this is last element in frame */
   bool last_elts;
   /** Index of element in frame */
@@ -153,6 +164,13 @@ typedef struct
   u8 started;
 } oct_crypto_main_t;
 
+static_always_inline bool
+oct_hw_ctx_cache_enable (void)
+{
+  return roc_errata_cpt_hang_on_mixed_ctx_val () ||
+        roc_model_is_cn10ka_b0 () || roc_model_is_cn10kb_a0 ();
+}
+
 extern oct_crypto_main_t oct_crypto_main;
 
 void oct_crypto_key_del_handler (vlib_main_t *vm,