drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #include <rte_security_driver.h>
  19 #include <rte_cycles.h>
  20 #include <rte_dev.h>
  21 #include <rte_kvargs.h>
  22 #include <rte_malloc.h>
  23 #include <rte_mbuf.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_string_fns.h>
  26
  27 #include <fsl_usd.h>
  28 #include <fsl_qman.h>
  29 #include <of.h>
  30
  31 /* RTA header files */
  32 #include <hw/desc/common.h>
  33 #include <hw/desc/algo.h>
  34 #include <hw/desc/ipsec.h>
  35
  36 #include <rte_dpaa_bus.h>
  37 #include <dpaa_sec.h>
  38 #include <dpaa_sec_log.h>
  39
  40 enum rta_sec_era rta_sec_era;
  41
  42 static uint8_t cryptodev_driver_id;
  43
  44 static __thread struct rte_crypto_op **dpaa_sec_ops;
  45 static __thread int dpaa_sec_op_nb;
  46
  47 static int
  48 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  49
  50 static inline void
  51 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  52 {
  53         if (!ctx->fd_status) {
  54                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  55         } else {
  56                 PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status);
  57                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  58         }
  59
  60         /* report op status to sym->op and then free the ctx memeory  */
  61         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  62 }
  63
  64 static inline struct dpaa_sec_op_ctx *
  65 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
  66 {
  67         struct dpaa_sec_op_ctx *ctx;
  68         int retval;
  69
  70         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  71         if (!ctx || retval) {
  72                 PMD_TX_LOG(ERR, "Alloc sec descriptor failed!");
  73                 return NULL;
  74         }
  75         /*
  76          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  77          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  78          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  79          * each packet, memset is costlier than dcbz_64().
  80          */
  81         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
  82         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
  83         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
  84         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
  85
  86         ctx->ctx_pool = ses->ctx_pool;
  87         ctx->vtop_offset = (uint64_t) ctx
  88                                 - rte_mempool_virt2iova(ctx);
  89
  90         return ctx;
  91 }
  92
  93 static inline rte_iova_t
  94 dpaa_mem_vtop(void *vaddr)
  95 {
  96         const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
  97         uint64_t vaddr_64, paddr;
  98         int i;
  99
 100         vaddr_64 = (uint64_t)vaddr;
 101         for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
 102                 if (vaddr_64 >= memseg[i].addr_64 &&
 103                     vaddr_64 < memseg[i].addr_64 + memseg[i].len) {
 104                         paddr = memseg[i].iova +
 105                                 (vaddr_64 - memseg[i].addr_64);
 106
 107                         return (rte_iova_t)paddr;
 108                 }
 109         }
 110         return (rte_iova_t)(NULL);
 111 }
 112
 113 /* virtual address conversin when mempool support is available for ctx */
 114 static inline phys_addr_t
 115 dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr)
 116 {
 117         return (uint64_t)vaddr - ctx->vtop_offset;
 118 }
 119
 120 static inline void *
 121 dpaa_mem_ptov(rte_iova_t paddr)
 122 {
 123         const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
 124         int i;
 125
 126         for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
 127                 if (paddr >= memseg[i].iova &&
 128                     (char *)paddr < (char *)memseg[i].iova + memseg[i].len)
 129                         return (void *)(memseg[i].addr_64 +
 130                                         (paddr - memseg[i].iova));
 131         }
 132         return NULL;
 133 }
 134
 135 static void
 136 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 137                    struct qman_fq *fq,
 138                    const struct qm_mr_entry *msg)
 139 {
 140         RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n",
 141                    fq->fqid, msg->ern.rc, msg->ern.seqnum);
 142 }
 143
 144 /* initialize the queue with dest chan as caam chan so that
 145  * all the packets in this queue could be dispatched into caam
 146  */
 147 static int
 148 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 149                  uint32_t fqid_out)
 150 {
 151         struct qm_mcc_initfq fq_opts;
 152         uint32_t flags;
 153         int ret = -1;
 154
 155         /* Clear FQ options */
 156         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 157
 158         flags = QMAN_INITFQ_FLAG_SCHED;
 159         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 160                           QM_INITFQ_WE_CONTEXTB;
 161
 162         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 163         fq_opts.fqd.context_b = fqid_out;
 164         fq_opts.fqd.dest.channel = qm_channel_caam;
 165         fq_opts.fqd.dest.wq = 0;
 166
 167         fq_in->cb.ern  = ern_sec_fq_handler;
 168
 169         PMD_INIT_LOG(DEBUG, "in-%x out-%x", fq_in->fqid, fqid_out);
 170
 171         ret = qman_init_fq(fq_in, flags, &fq_opts);
 172         if (unlikely(ret != 0))
 173                 PMD_INIT_LOG(ERR, "qman_init_fq failed %d", ret);
 174
 175         return ret;
 176 }
 177
 178 /* something is put into in_fq and caam put the crypto result into out_fq */
 179 static enum qman_cb_dqrr_result
 180 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 181                   struct qman_fq *fq __always_unused,
 182                   const struct qm_dqrr_entry *dqrr)
 183 {
 184         const struct qm_fd *fd;
 185         struct dpaa_sec_job *job;
 186         struct dpaa_sec_op_ctx *ctx;
 187
 188         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
 189                 return qman_cb_dqrr_defer;
 190
 191         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 192                 return qman_cb_dqrr_consume;
 193
 194         fd = &dqrr->fd;
 195         /* sg is embedded in an op ctx,
 196          * sg[0] is for output
 197          * sg[1] for input
 198          */
 199         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 200
 201         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 202         ctx->fd_status = fd->status;
 203         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 204                 struct qm_sg_entry *sg_out;
 205                 uint32_t len;
 206
 207                 sg_out = &job->sg[0];
 208                 hw_sg_to_cpu(sg_out);
 209                 len = sg_out->length;
 210                 ctx->op->sym->m_src->pkt_len = len;
 211                 ctx->op->sym->m_src->data_len = len;
 212         }
 213         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
 214         dpaa_sec_op_ending(ctx);
 215
 216         return qman_cb_dqrr_consume;
 217 }
 218
 219 /* caam result is put into this queue */
 220 static int
 221 dpaa_sec_init_tx(struct qman_fq *fq)
 222 {
 223         int ret;
 224         struct qm_mcc_initfq opts;
 225         uint32_t flags;
 226
 227         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 228                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 229
 230         ret = qman_create_fq(0, flags, fq);
 231         if (unlikely(ret)) {
 232                 PMD_INIT_LOG(ERR, "qman_create_fq failed");
 233                 return ret;
 234         }
 235
 236         memset(&opts, 0, sizeof(opts));
 237         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 238                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 239
 240         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 241
 242         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 243         fq->cb.ern  = ern_sec_fq_handler;
 244
 245         ret = qman_init_fq(fq, 0, &opts);
 246         if (unlikely(ret)) {
 247                 PMD_INIT_LOG(ERR, "unable to init caam source fq!");
 248                 return ret;
 249         }
 250
 251         return ret;
 252 }
 253
 254 static inline int is_cipher_only(dpaa_sec_session *ses)
 255 {
 256         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 257                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 258 }
 259
 260 static inline int is_auth_only(dpaa_sec_session *ses)
 261 {
 262         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 263                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 264 }
 265
 266 static inline int is_aead(dpaa_sec_session *ses)
 267 {
 268         return ((ses->cipher_alg == 0) &&
 269                 (ses->auth_alg == 0) &&
 270                 (ses->aead_alg != 0));
 271 }
 272
 273 static inline int is_auth_cipher(dpaa_sec_session *ses)
 274 {
 275         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 276                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 277                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 278 }
 279
 280 static inline int is_proto_ipsec(dpaa_sec_session *ses)
 281 {
 282         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 283 }
 284
 285 static inline int is_encode(dpaa_sec_session *ses)
 286 {
 287         return ses->dir == DIR_ENC;
 288 }
 289
 290 static inline int is_decode(dpaa_sec_session *ses)
 291 {
 292         return ses->dir == DIR_DEC;
 293 }
 294
 295 static inline void
 296 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 297 {
 298         switch (ses->auth_alg) {
 299         case RTE_CRYPTO_AUTH_NULL:
 300                 ses->digest_length = 0;
 301                 break;
 302         case RTE_CRYPTO_AUTH_MD5_HMAC:
 303                 alginfo_a->algtype =
 304                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 305                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 306                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 307                 break;
 308         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 309                 alginfo_a->algtype =
 310                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 311                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 312                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 313                 break;
 314         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 315                 alginfo_a->algtype =
 316                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 317                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 318                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 319                 break;
 320         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 321                 alginfo_a->algtype =
 322                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 323                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 324                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 325                 break;
 326         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 327                 alginfo_a->algtype =
 328                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 329                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 330                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 331                 break;
 332         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 333                 alginfo_a->algtype =
 334                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 335                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 336                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 337                 break;
 338         default:
 339                 PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg);
 340         }
 341 }
 342
 343 static inline void
 344 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 345 {
 346         switch (ses->cipher_alg) {
 347         case RTE_CRYPTO_CIPHER_NULL:
 348                 break;
 349         case RTE_CRYPTO_CIPHER_AES_CBC:
 350                 alginfo_c->algtype =
 351                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 352                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 353                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 354                 break;
 355         case RTE_CRYPTO_CIPHER_3DES_CBC:
 356                 alginfo_c->algtype =
 357                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 358                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 359                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 360                 break;
 361         case RTE_CRYPTO_CIPHER_AES_CTR:
 362                 alginfo_c->algtype =
 363                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 364                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 365                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 366                 break;
 367         default:
 368                 PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg);
 369         }
 370 }
 371
 372 static inline void
 373 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 374 {
 375         switch (ses->aead_alg) {
 376         case RTE_CRYPTO_AEAD_AES_GCM:
 377                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 378                 alginfo->algmode = OP_ALG_AAI_GCM;
 379                 break;
 380         default:
 381                 PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg);
 382         }
 383 }
 384
 385
 386 /* prepare command block of the session */
 387 static int
 388 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 389 {
 390         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 391         uint32_t shared_desc_len = 0;
 392         struct sec_cdb *cdb = &ses->cdb;
 393         int err;
 394 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 395         int swap = false;
 396 #else
 397         int swap = true;
 398 #endif
 399
 400         memset(cdb, 0, sizeof(struct sec_cdb));
 401
 402         if (is_cipher_only(ses)) {
 403                 caam_cipher_alg(ses, &alginfo_c);
 404                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 405                         PMD_TX_LOG(ERR, "not supported cipher alg\n");
 406                         return -ENOTSUP;
 407                 }
 408
 409                 alginfo_c.key = (uint64_t)ses->cipher_key.data;
 410                 alginfo_c.keylen = ses->cipher_key.length;
 411                 alginfo_c.key_enc_flags = 0;
 412                 alginfo_c.key_type = RTA_DATA_IMM;
 413
 414                 shared_desc_len = cnstr_shdsc_blkcipher(
 415                                                 cdb->sh_desc, true,
 416                                                 swap, &alginfo_c,
 417                                                 NULL,
 418                                                 ses->iv.length,
 419                                                 ses->dir);
 420         } else if (is_auth_only(ses)) {
 421                 caam_auth_alg(ses, &alginfo_a);
 422                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 423                         PMD_TX_LOG(ERR, "not supported auth alg\n");
 424                         return -ENOTSUP;
 425                 }
 426
 427                 alginfo_a.key = (uint64_t)ses->auth_key.data;
 428                 alginfo_a.keylen = ses->auth_key.length;
 429                 alginfo_a.key_enc_flags = 0;
 430                 alginfo_a.key_type = RTA_DATA_IMM;
 431
 432                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 433                                                    swap, &alginfo_a,
 434                                                    !ses->dir,
 435                                                    ses->digest_length);
 436         } else if (is_aead(ses)) {
 437                 caam_aead_alg(ses, &alginfo);
 438                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 439                         PMD_TX_LOG(ERR, "not supported aead alg\n");
 440                         return -ENOTSUP;
 441                 }
 442                 alginfo.key = (uint64_t)ses->aead_key.data;
 443                 alginfo.keylen = ses->aead_key.length;
 444                 alginfo.key_enc_flags = 0;
 445                 alginfo.key_type = RTA_DATA_IMM;
 446
 447                 if (ses->dir == DIR_ENC)
 448                         shared_desc_len = cnstr_shdsc_gcm_encap(
 449                                         cdb->sh_desc, true, swap,
 450                                         &alginfo,
 451                                         ses->iv.length,
 452                                         ses->digest_length);
 453                 else
 454                         shared_desc_len = cnstr_shdsc_gcm_decap(
 455                                         cdb->sh_desc, true, swap,
 456                                         &alginfo,
 457                                         ses->iv.length,
 458                                         ses->digest_length);
 459         } else {
 460                 caam_cipher_alg(ses, &alginfo_c);
 461                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 462                         PMD_TX_LOG(ERR, "not supported cipher alg\n");
 463                         return -ENOTSUP;
 464                 }
 465
 466                 alginfo_c.key = (uint64_t)ses->cipher_key.data;
 467                 alginfo_c.keylen = ses->cipher_key.length;
 468                 alginfo_c.key_enc_flags = 0;
 469                 alginfo_c.key_type = RTA_DATA_IMM;
 470
 471                 caam_auth_alg(ses, &alginfo_a);
 472                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 473                         PMD_TX_LOG(ERR, "not supported auth alg\n");
 474                         return -ENOTSUP;
 475                 }
 476
 477                 alginfo_a.key = (uint64_t)ses->auth_key.data;
 478                 alginfo_a.keylen = ses->auth_key.length;
 479                 alginfo_a.key_enc_flags = 0;
 480                 alginfo_a.key_type = RTA_DATA_IMM;
 481
 482                 cdb->sh_desc[0] = alginfo_c.keylen;
 483                 cdb->sh_desc[1] = alginfo_a.keylen;
 484                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 485                                        MIN_JOB_DESC_SIZE,
 486                                        (unsigned int *)cdb->sh_desc,
 487                                        &cdb->sh_desc[2], 2);
 488
 489                 if (err < 0) {
 490                         PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths");
 491                         return err;
 492                 }
 493                 if (cdb->sh_desc[2] & 1)
 494                         alginfo_c.key_type = RTA_DATA_IMM;
 495                 else {
 496                         alginfo_c.key = (uint64_t)dpaa_mem_vtop(
 497                                                         (void *)alginfo_c.key);
 498                         alginfo_c.key_type = RTA_DATA_PTR;
 499                 }
 500                 if (cdb->sh_desc[2] & (1<<1))
 501                         alginfo_a.key_type = RTA_DATA_IMM;
 502                 else {
 503                         alginfo_a.key = (uint64_t)dpaa_mem_vtop(
 504                                                         (void *)alginfo_a.key);
 505                         alginfo_a.key_type = RTA_DATA_PTR;
 506                 }
 507                 cdb->sh_desc[0] = 0;
 508                 cdb->sh_desc[1] = 0;
 509                 cdb->sh_desc[2] = 0;
 510                 if (is_proto_ipsec(ses)) {
 511                         if (ses->dir == DIR_ENC) {
 512                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 513                                                 cdb->sh_desc,
 514                                                 true, swap, &ses->encap_pdb,
 515                                                 (uint8_t *)&ses->ip4_hdr,
 516                                                 &alginfo_c, &alginfo_a);
 517                         } else if (ses->dir == DIR_DEC) {
 518                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 519                                                 cdb->sh_desc,
 520                                                 true, swap, &ses->decap_pdb,
 521                                                 &alginfo_c, &alginfo_a);
 522                         }
 523                 } else {
 524                         /* Auth_only_len is set as 0 here and it will be
 525                          * overwritten in fd for each packet.
 526                          */
 527                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 528                                         true, swap, &alginfo_c, &alginfo_a,
 529                                         ses->iv.length, 0,
 530                                         ses->digest_length, ses->dir);
 531                 }
 532         }
 533         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 534         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 535         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 536
 537         return 0;
 538 }
 539
 540 /* qp is lockless, should be accessed by only one thread */
 541 static int
 542 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 543 {
 544         struct qman_fq *fq;
 545         unsigned int pkts = 0;
 546         int ret;
 547         struct qm_dqrr_entry *dq;
 548
 549         fq = &qp->outq;
 550         ret = qman_set_vdq(fq, (nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES) ?
 551                                 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_ops);
 552         if (ret)
 553                 return 0;
 554
 555         do {
 556                 const struct qm_fd *fd;
 557                 struct dpaa_sec_job *job;
 558                 struct dpaa_sec_op_ctx *ctx;
 559                 struct rte_crypto_op *op;
 560
 561                 dq = qman_dequeue(fq);
 562                 if (!dq)
 563                         continue;
 564
 565                 fd = &dq->fd;
 566                 /* sg is embedded in an op ctx,
 567                  * sg[0] is for output
 568                  * sg[1] for input
 569                  */
 570                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 571
 572                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 573                 ctx->fd_status = fd->status;
 574                 op = ctx->op;
 575                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 576                         struct qm_sg_entry *sg_out;
 577                         uint32_t len;
 578
 579                         sg_out = &job->sg[0];
 580                         hw_sg_to_cpu(sg_out);
 581                         len = sg_out->length;
 582                         op->sym->m_src->pkt_len = len;
 583                         op->sym->m_src->data_len = len;
 584                 }
 585                 if (!ctx->fd_status) {
 586                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 587                 } else {
 588                         printf("\nSEC return err: 0x%x", ctx->fd_status);
 589                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 590                 }
 591                 ops[pkts++] = op;
 592
 593                 /* report op status to sym->op and then free the ctx memeory */
 594                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 595
 596                 qman_dqrr_consume(fq, dq);
 597         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 598
 599         return pkts;
 600 }
 601
 602 static inline struct dpaa_sec_job *
 603 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 604 {
 605         struct rte_crypto_sym_op *sym = op->sym;
 606         struct rte_mbuf *mbuf = sym->m_src;
 607         struct dpaa_sec_job *cf;
 608         struct dpaa_sec_op_ctx *ctx;
 609         struct qm_sg_entry *sg, *out_sg, *in_sg;
 610         phys_addr_t start_addr;
 611         uint8_t *old_digest, extra_segs;
 612
 613         if (is_decode(ses))
 614                 extra_segs = 3;
 615         else
 616                 extra_segs = 2;
 617
 618         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 619                 PMD_TX_LOG(ERR, "Auth: Max sec segs supported is %d\n",
 620                                                                 MAX_SG_ENTRIES);
 621                 return NULL;
 622         }
 623         ctx = dpaa_sec_alloc_ctx(ses);
 624         if (!ctx)
 625                 return NULL;
 626
 627         cf = &ctx->job;
 628         ctx->op = op;
 629         old_digest = ctx->digest;
 630
 631         /* output */
 632         out_sg = &cf->sg[0];
 633         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 634         out_sg->length = ses->digest_length;
 635         cpu_to_hw_sg(out_sg);
 636
 637         /* input */
 638         in_sg = &cf->sg[1];
 639         /* need to extend the input to a compound frame */
 640         in_sg->extension = 1;
 641         in_sg->final = 1;
 642         in_sg->length = sym->auth.data.length;
 643         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 644
 645         /* 1st seg */
 646         sg = in_sg + 1;
 647         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 648         sg->length = mbuf->data_len - sym->auth.data.offset;
 649         sg->offset = sym->auth.data.offset;
 650
 651         /* Successive segs */
 652         mbuf = mbuf->next;
 653         while (mbuf) {
 654                 cpu_to_hw_sg(sg);
 655                 sg++;
 656                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 657                 sg->length = mbuf->data_len;
 658                 mbuf = mbuf->next;
 659         }
 660
 661         if (is_decode(ses)) {
 662                 /* Digest verification case */
 663                 cpu_to_hw_sg(sg);
 664                 sg++;
 665                 rte_memcpy(old_digest, sym->auth.digest.data,
 666                                 ses->digest_length);
 667                 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest);
 668                 qm_sg_entry_set64(sg, start_addr);
 669                 sg->length = ses->digest_length;
 670                 in_sg->length += ses->digest_length;
 671         } else {
 672                 /* Digest calculation case */
 673                 sg->length -= ses->digest_length;
 674         }
 675         sg->final = 1;
 676         cpu_to_hw_sg(sg);
 677         cpu_to_hw_sg(in_sg);
 678
 679         return cf;
 680 }
 681
 682 /**
 683  * packet looks like:
 684  *              |<----data_len------->|
 685  *    |ip_header|ah_header|icv|payload|
 686  *              ^
 687  *              |
 688  *         mbuf->pkt.data
 689  */
 690 static inline struct dpaa_sec_job *
 691 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 692 {
 693         struct rte_crypto_sym_op *sym = op->sym;
 694         struct rte_mbuf *mbuf = sym->m_src;
 695         struct dpaa_sec_job *cf;
 696         struct dpaa_sec_op_ctx *ctx;
 697         struct qm_sg_entry *sg;
 698         rte_iova_t start_addr;
 699         uint8_t *old_digest;
 700
 701         ctx = dpaa_sec_alloc_ctx(ses);
 702         if (!ctx)
 703                 return NULL;
 704
 705         cf = &ctx->job;
 706         ctx->op = op;
 707         old_digest = ctx->digest;
 708
 709         start_addr = rte_pktmbuf_iova(mbuf);
 710         /* output */
 711         sg = &cf->sg[0];
 712         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 713         sg->length = ses->digest_length;
 714         cpu_to_hw_sg(sg);
 715
 716         /* input */
 717         sg = &cf->sg[1];
 718         if (is_decode(ses)) {
 719                 /* need to extend the input to a compound frame */
 720                 sg->extension = 1;
 721                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 722                 sg->length = sym->auth.data.length + ses->digest_length;
 723                 sg->final = 1;
 724                 cpu_to_hw_sg(sg);
 725
 726                 sg = &cf->sg[2];
 727                 /* hash result or digest, save digest first */
 728                 rte_memcpy(old_digest, sym->auth.digest.data,
 729                            ses->digest_length);
 730                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 731                 sg->length = sym->auth.data.length;
 732                 cpu_to_hw_sg(sg);
 733
 734                 /* let's check digest by hw */
 735                 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest);
 736                 sg++;
 737                 qm_sg_entry_set64(sg, start_addr);
 738                 sg->length = ses->digest_length;
 739                 sg->final = 1;
 740                 cpu_to_hw_sg(sg);
 741         } else {
 742                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 743                 sg->length = sym->auth.data.length;
 744                 sg->final = 1;
 745                 cpu_to_hw_sg(sg);
 746         }
 747
 748         return cf;
 749 }
 750
 751 static inline struct dpaa_sec_job *
 752 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 753 {
 754         struct rte_crypto_sym_op *sym = op->sym;
 755         struct dpaa_sec_job *cf;
 756         struct dpaa_sec_op_ctx *ctx;
 757         struct qm_sg_entry *sg, *out_sg, *in_sg;
 758         struct rte_mbuf *mbuf;
 759         uint8_t req_segs;
 760         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 761                         ses->iv.offset);
 762
 763         if (sym->m_dst) {
 764                 mbuf = sym->m_dst;
 765                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 766         } else {
 767                 mbuf = sym->m_src;
 768                 req_segs = mbuf->nb_segs * 2 + 3;
 769         }
 770
 771         if (req_segs > MAX_SG_ENTRIES) {
 772                 PMD_TX_LOG(ERR, "Cipher: Max sec segs supported is %d\n",
 773                                                                 MAX_SG_ENTRIES);
 774                 return NULL;
 775         }
 776
 777         ctx = dpaa_sec_alloc_ctx(ses);
 778         if (!ctx)
 779                 return NULL;
 780
 781         cf = &ctx->job;
 782         ctx->op = op;
 783
 784         /* output */
 785         out_sg = &cf->sg[0];
 786         out_sg->extension = 1;
 787         out_sg->length = sym->cipher.data.length;
 788         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 789         cpu_to_hw_sg(out_sg);
 790
 791         /* 1st seg */
 792         sg = &cf->sg[2];
 793         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 794         sg->length = mbuf->data_len - sym->cipher.data.offset;
 795         sg->offset = sym->cipher.data.offset;
 796
 797         /* Successive segs */
 798         mbuf = mbuf->next;
 799         while (mbuf) {
 800                 cpu_to_hw_sg(sg);
 801                 sg++;
 802                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 803                 sg->length = mbuf->data_len;
 804                 mbuf = mbuf->next;
 805         }
 806         sg->final = 1;
 807         cpu_to_hw_sg(sg);
 808
 809         /* input */
 810         mbuf = sym->m_src;
 811         in_sg = &cf->sg[1];
 812         in_sg->extension = 1;
 813         in_sg->final = 1;
 814         in_sg->length = sym->cipher.data.length + ses->iv.length;
 815
 816         sg++;
 817         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
 818         cpu_to_hw_sg(in_sg);
 819
 820         /* IV */
 821         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 822         sg->length = ses->iv.length;
 823         cpu_to_hw_sg(sg);
 824
 825         /* 1st seg */
 826         sg++;
 827         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 828         sg->length = mbuf->data_len - sym->cipher.data.offset;
 829         sg->offset = sym->cipher.data.offset;
 830
 831         /* Successive segs */
 832         mbuf = mbuf->next;
 833         while (mbuf) {
 834                 cpu_to_hw_sg(sg);
 835                 sg++;
 836                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 837                 sg->length = mbuf->data_len;
 838                 mbuf = mbuf->next;
 839         }
 840         sg->final = 1;
 841         cpu_to_hw_sg(sg);
 842
 843         return cf;
 844 }
 845
 846 static inline struct dpaa_sec_job *
 847 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 848 {
 849         struct rte_crypto_sym_op *sym = op->sym;
 850         struct dpaa_sec_job *cf;
 851         struct dpaa_sec_op_ctx *ctx;
 852         struct qm_sg_entry *sg;
 853         rte_iova_t src_start_addr, dst_start_addr;
 854         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 855                         ses->iv.offset);
 856
 857         ctx = dpaa_sec_alloc_ctx(ses);
 858         if (!ctx)
 859                 return NULL;
 860
 861         cf = &ctx->job;
 862         ctx->op = op;
 863
 864         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 865
 866         if (sym->m_dst)
 867                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 868         else
 869                 dst_start_addr = src_start_addr;
 870
 871         /* output */
 872         sg = &cf->sg[0];
 873         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
 874         sg->length = sym->cipher.data.length + ses->iv.length;
 875         cpu_to_hw_sg(sg);
 876
 877         /* input */
 878         sg = &cf->sg[1];
 879
 880         /* need to extend the input to a compound frame */
 881         sg->extension = 1;
 882         sg->final = 1;
 883         sg->length = sym->cipher.data.length + ses->iv.length;
 884         qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 885         cpu_to_hw_sg(sg);
 886
 887         sg = &cf->sg[2];
 888         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 889         sg->length = ses->iv.length;
 890         cpu_to_hw_sg(sg);
 891
 892         sg++;
 893         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
 894         sg->length = sym->cipher.data.length;
 895         sg->final = 1;
 896         cpu_to_hw_sg(sg);
 897
 898         return cf;
 899 }
 900
 901 static inline struct dpaa_sec_job *
 902 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 903 {
 904         struct rte_crypto_sym_op *sym = op->sym;
 905         struct dpaa_sec_job *cf;
 906         struct dpaa_sec_op_ctx *ctx;
 907         struct qm_sg_entry *sg, *out_sg, *in_sg;
 908         struct rte_mbuf *mbuf;
 909         uint8_t req_segs;
 910         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 911                         ses->iv.offset);
 912
 913         if (sym->m_dst) {
 914                 mbuf = sym->m_dst;
 915                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
 916         } else {
 917                 mbuf = sym->m_src;
 918                 req_segs = mbuf->nb_segs * 2 + 4;
 919         }
 920
 921         if (ses->auth_only_len)
 922                 req_segs++;
 923
 924         if (req_segs > MAX_SG_ENTRIES) {
 925                 PMD_TX_LOG(ERR, "AEAD: Max sec segs supported is %d\n",
 926                                 MAX_SG_ENTRIES);
 927                 return NULL;
 928         }
 929
 930         ctx = dpaa_sec_alloc_ctx(ses);
 931         if (!ctx)
 932                 return NULL;
 933
 934         cf = &ctx->job;
 935         ctx->op = op;
 936
 937         rte_prefetch0(cf->sg);
 938
 939         /* output */
 940         out_sg = &cf->sg[0];
 941         out_sg->extension = 1;
 942         if (is_encode(ses))
 943                 out_sg->length = sym->aead.data.length + ses->auth_only_len
 944                                                 + ses->digest_length;
 945         else
 946                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
 947
 948         /* output sg entries */
 949         sg = &cf->sg[2];
 950         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg));
 951         cpu_to_hw_sg(out_sg);
 952
 953         /* 1st seg */
 954         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 955         sg->length = mbuf->data_len - sym->aead.data.offset +
 956                                         ses->auth_only_len;
 957         sg->offset = sym->aead.data.offset - ses->auth_only_len;
 958
 959         /* Successive segs */
 960         mbuf = mbuf->next;
 961         while (mbuf) {
 962                 cpu_to_hw_sg(sg);
 963                 sg++;
 964                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 965                 sg->length = mbuf->data_len;
 966                 mbuf = mbuf->next;
 967         }
 968         sg->length -= ses->digest_length;
 969
 970         if (is_encode(ses)) {
 971                 cpu_to_hw_sg(sg);
 972                 /* set auth output */
 973                 sg++;
 974                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
 975                 sg->length = ses->digest_length;
 976         }
 977         sg->final = 1;
 978         cpu_to_hw_sg(sg);
 979
 980         /* input */
 981         mbuf = sym->m_src;
 982         in_sg = &cf->sg[1];
 983         in_sg->extension = 1;
 984         in_sg->final = 1;
 985         if (is_encode(ses))
 986                 in_sg->length = ses->iv.length + sym->aead.data.length
 987                                                         + ses->auth_only_len;
 988         else
 989                 in_sg->length = ses->iv.length + sym->aead.data.length
 990                                 + ses->auth_only_len + ses->digest_length;
 991
 992         /* input sg entries */
 993         sg++;
 994         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
 995         cpu_to_hw_sg(in_sg);
 996
 997         /* 1st seg IV */
 998         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 999         sg->length = ses->iv.length;
1000         cpu_to_hw_sg(sg);
1001
1002         /* 2nd seg auth only */
1003         if (ses->auth_only_len) {
1004                 sg++;
1005                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
1006                 sg->length = ses->auth_only_len;
1007                 cpu_to_hw_sg(sg);
1008         }
1009
1010         /* 3rd seg */
1011         sg++;
1012         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1013         sg->length = mbuf->data_len - sym->aead.data.offset;
1014         sg->offset = sym->aead.data.offset;
1015
1016         /* Successive segs */
1017         mbuf = mbuf->next;
1018         while (mbuf) {
1019                 cpu_to_hw_sg(sg);
1020                 sg++;
1021                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1022                 sg->length = mbuf->data_len;
1023                 mbuf = mbuf->next;
1024         }
1025
1026         if (is_decode(ses)) {
1027                 cpu_to_hw_sg(sg);
1028                 sg++;
1029                 memcpy(ctx->digest, sym->aead.digest.data,
1030                         ses->digest_length);
1031                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1032                 sg->length = ses->digest_length;
1033         }
1034         sg->final = 1;
1035         cpu_to_hw_sg(sg);
1036
1037         return cf;
1038 }
1039
1040 static inline struct dpaa_sec_job *
1041 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1042 {
1043         struct rte_crypto_sym_op *sym = op->sym;
1044         struct dpaa_sec_job *cf;
1045         struct dpaa_sec_op_ctx *ctx;
1046         struct qm_sg_entry *sg;
1047         uint32_t length = 0;
1048         rte_iova_t src_start_addr, dst_start_addr;
1049         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1050                         ses->iv.offset);
1051
1052         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1053
1054         if (sym->m_dst)
1055                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1056         else
1057                 dst_start_addr = src_start_addr;
1058
1059         ctx = dpaa_sec_alloc_ctx(ses);
1060         if (!ctx)
1061                 return NULL;
1062
1063         cf = &ctx->job;
1064         ctx->op = op;
1065
1066         /* input */
1067         rte_prefetch0(cf->sg);
1068         sg = &cf->sg[2];
1069         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
1070         if (is_encode(ses)) {
1071                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1072                 sg->length = ses->iv.length;
1073                 length += sg->length;
1074                 cpu_to_hw_sg(sg);
1075
1076                 sg++;
1077                 if (ses->auth_only_len) {
1078                         qm_sg_entry_set64(sg,
1079                                           dpaa_mem_vtop(sym->aead.aad.data));
1080                         sg->length = ses->auth_only_len;
1081                         length += sg->length;
1082                         cpu_to_hw_sg(sg);
1083                         sg++;
1084                 }
1085                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1086                 sg->length = sym->aead.data.length;
1087                 length += sg->length;
1088                 sg->final = 1;
1089                 cpu_to_hw_sg(sg);
1090         } else {
1091                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1092                 sg->length = ses->iv.length;
1093                 length += sg->length;
1094                 cpu_to_hw_sg(sg);
1095
1096                 sg++;
1097                 if (ses->auth_only_len) {
1098                         qm_sg_entry_set64(sg,
1099                                           dpaa_mem_vtop(sym->aead.aad.data));
1100                         sg->length = ses->auth_only_len;
1101                         length += sg->length;
1102                         cpu_to_hw_sg(sg);
1103                         sg++;
1104                 }
1105                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1106                 sg->length = sym->aead.data.length;
1107                 length += sg->length;
1108                 cpu_to_hw_sg(sg);
1109
1110                 memcpy(ctx->digest, sym->aead.digest.data,
1111                        ses->digest_length);
1112                 sg++;
1113
1114                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1115                 sg->length = ses->digest_length;
1116                 length += sg->length;
1117                 sg->final = 1;
1118                 cpu_to_hw_sg(sg);
1119         }
1120         /* input compound frame */
1121         cf->sg[1].length = length;
1122         cf->sg[1].extension = 1;
1123         cf->sg[1].final = 1;
1124         cpu_to_hw_sg(&cf->sg[1]);
1125
1126         /* output */
1127         sg++;
1128         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
1129         qm_sg_entry_set64(sg,
1130                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1131         sg->length = sym->aead.data.length + ses->auth_only_len;
1132         length = sg->length;
1133         if (is_encode(ses)) {
1134                 cpu_to_hw_sg(sg);
1135                 /* set auth output */
1136                 sg++;
1137                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1138                 sg->length = ses->digest_length;
1139                 length += sg->length;
1140         }
1141         sg->final = 1;
1142         cpu_to_hw_sg(sg);
1143
1144         /* output compound frame */
1145         cf->sg[0].length = length;
1146         cf->sg[0].extension = 1;
1147         cpu_to_hw_sg(&cf->sg[0]);
1148
1149         return cf;
1150 }
1151
1152 static inline struct dpaa_sec_job *
1153 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1154 {
1155         struct rte_crypto_sym_op *sym = op->sym;
1156         struct dpaa_sec_job *cf;
1157         struct dpaa_sec_op_ctx *ctx;
1158         struct qm_sg_entry *sg, *out_sg, *in_sg;
1159         struct rte_mbuf *mbuf;
1160         uint8_t req_segs;
1161         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1162                         ses->iv.offset);
1163
1164         if (sym->m_dst) {
1165                 mbuf = sym->m_dst;
1166                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1167         } else {
1168                 mbuf = sym->m_src;
1169                 req_segs = mbuf->nb_segs * 2 + 4;
1170         }
1171
1172         if (req_segs > MAX_SG_ENTRIES) {
1173                 PMD_TX_LOG(ERR, "Cipher-Auth: Max sec segs supported is %d\n",
1174                                 MAX_SG_ENTRIES);
1175                 return NULL;
1176         }
1177
1178         ctx = dpaa_sec_alloc_ctx(ses);
1179         if (!ctx)
1180                 return NULL;
1181
1182         cf = &ctx->job;
1183         ctx->op = op;
1184
1185         rte_prefetch0(cf->sg);
1186
1187         /* output */
1188         out_sg = &cf->sg[0];
1189         out_sg->extension = 1;
1190         if (is_encode(ses))
1191                 out_sg->length = sym->auth.data.length + ses->digest_length;
1192         else
1193                 out_sg->length = sym->auth.data.length;
1194
1195         /* output sg entries */
1196         sg = &cf->sg[2];
1197         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg));
1198         cpu_to_hw_sg(out_sg);
1199
1200         /* 1st seg */
1201         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1202         sg->length = mbuf->data_len - sym->auth.data.offset;
1203         sg->offset = sym->auth.data.offset;
1204
1205         /* Successive segs */
1206         mbuf = mbuf->next;
1207         while (mbuf) {
1208                 cpu_to_hw_sg(sg);
1209                 sg++;
1210                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1211                 sg->length = mbuf->data_len;
1212                 mbuf = mbuf->next;
1213         }
1214         sg->length -= ses->digest_length;
1215
1216         if (is_encode(ses)) {
1217                 cpu_to_hw_sg(sg);
1218                 /* set auth output */
1219                 sg++;
1220                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1221                 sg->length = ses->digest_length;
1222         }
1223         sg->final = 1;
1224         cpu_to_hw_sg(sg);
1225
1226         /* input */
1227         mbuf = sym->m_src;
1228         in_sg = &cf->sg[1];
1229         in_sg->extension = 1;
1230         in_sg->final = 1;
1231         if (is_encode(ses))
1232                 in_sg->length = ses->iv.length + sym->auth.data.length;
1233         else
1234                 in_sg->length = ses->iv.length + sym->auth.data.length
1235                                                 + ses->digest_length;
1236
1237         /* input sg entries */
1238         sg++;
1239         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
1240         cpu_to_hw_sg(in_sg);
1241
1242         /* 1st seg IV */
1243         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1244         sg->length = ses->iv.length;
1245         cpu_to_hw_sg(sg);
1246
1247         /* 2nd seg */
1248         sg++;
1249         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1250         sg->length = mbuf->data_len - sym->auth.data.offset;
1251         sg->offset = sym->auth.data.offset;
1252
1253         /* Successive segs */
1254         mbuf = mbuf->next;
1255         while (mbuf) {
1256                 cpu_to_hw_sg(sg);
1257                 sg++;
1258                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1259                 sg->length = mbuf->data_len;
1260                 mbuf = mbuf->next;
1261         }
1262
1263         sg->length -= ses->digest_length;
1264         if (is_decode(ses)) {
1265                 cpu_to_hw_sg(sg);
1266                 sg++;
1267                 memcpy(ctx->digest, sym->auth.digest.data,
1268                         ses->digest_length);
1269                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1270                 sg->length = ses->digest_length;
1271         }
1272         sg->final = 1;
1273         cpu_to_hw_sg(sg);
1274
1275         return cf;
1276 }
1277
1278 static inline struct dpaa_sec_job *
1279 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1280 {
1281         struct rte_crypto_sym_op *sym = op->sym;
1282         struct dpaa_sec_job *cf;
1283         struct dpaa_sec_op_ctx *ctx;
1284         struct qm_sg_entry *sg;
1285         rte_iova_t src_start_addr, dst_start_addr;
1286         uint32_t length = 0;
1287         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1288                         ses->iv.offset);
1289
1290         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1291         if (sym->m_dst)
1292                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1293         else
1294                 dst_start_addr = src_start_addr;
1295
1296         ctx = dpaa_sec_alloc_ctx(ses);
1297         if (!ctx)
1298                 return NULL;
1299
1300         cf = &ctx->job;
1301         ctx->op = op;
1302
1303         /* input */
1304         rte_prefetch0(cf->sg);
1305         sg = &cf->sg[2];
1306         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
1307         if (is_encode(ses)) {
1308                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1309                 sg->length = ses->iv.length;
1310                 length += sg->length;
1311                 cpu_to_hw_sg(sg);
1312
1313                 sg++;
1314                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1315                 sg->length = sym->auth.data.length;
1316                 length += sg->length;
1317                 sg->final = 1;
1318                 cpu_to_hw_sg(sg);
1319         } else {
1320                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1321                 sg->length = ses->iv.length;
1322                 length += sg->length;
1323                 cpu_to_hw_sg(sg);
1324
1325                 sg++;
1326
1327                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1328                 sg->length = sym->auth.data.length;
1329                 length += sg->length;
1330                 cpu_to_hw_sg(sg);
1331
1332                 memcpy(ctx->digest, sym->auth.digest.data,
1333                        ses->digest_length);
1334                 sg++;
1335
1336                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1337                 sg->length = ses->digest_length;
1338                 length += sg->length;
1339                 sg->final = 1;
1340                 cpu_to_hw_sg(sg);
1341         }
1342         /* input compound frame */
1343         cf->sg[1].length = length;
1344         cf->sg[1].extension = 1;
1345         cf->sg[1].final = 1;
1346         cpu_to_hw_sg(&cf->sg[1]);
1347
1348         /* output */
1349         sg++;
1350         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
1351         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1352         sg->length = sym->cipher.data.length;
1353         length = sg->length;
1354         if (is_encode(ses)) {
1355                 cpu_to_hw_sg(sg);
1356                 /* set auth output */
1357                 sg++;
1358                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1359                 sg->length = ses->digest_length;
1360                 length += sg->length;
1361         }
1362         sg->final = 1;
1363         cpu_to_hw_sg(sg);
1364
1365         /* output compound frame */
1366         cf->sg[0].length = length;
1367         cf->sg[0].extension = 1;
1368         cpu_to_hw_sg(&cf->sg[0]);
1369
1370         return cf;
1371 }
1372
1373 static inline struct dpaa_sec_job *
1374 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1375 {
1376         struct rte_crypto_sym_op *sym = op->sym;
1377         struct dpaa_sec_job *cf;
1378         struct dpaa_sec_op_ctx *ctx;
1379         struct qm_sg_entry *sg;
1380         phys_addr_t src_start_addr, dst_start_addr;
1381
1382         ctx = dpaa_sec_alloc_ctx(ses);
1383         if (!ctx)
1384                 return NULL;
1385         cf = &ctx->job;
1386         ctx->op = op;
1387
1388         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1389
1390         if (sym->m_dst)
1391                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1392         else
1393                 dst_start_addr = src_start_addr;
1394
1395         /* input */
1396         sg = &cf->sg[1];
1397         qm_sg_entry_set64(sg, src_start_addr);
1398         sg->length = sym->m_src->pkt_len;
1399         sg->final = 1;
1400         cpu_to_hw_sg(sg);
1401
1402         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1403         /* output */
1404         sg = &cf->sg[0];
1405         qm_sg_entry_set64(sg, dst_start_addr);
1406         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1407         cpu_to_hw_sg(sg);
1408
1409         return cf;
1410 }
1411
1412 static uint16_t
1413 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1414                        uint16_t nb_ops)
1415 {
1416         /* Function to transmit the frames to given device and queuepair */
1417         uint32_t loop;
1418         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1419         uint16_t num_tx = 0;
1420         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1421         uint32_t frames_to_send;
1422         struct rte_crypto_op *op;
1423         struct dpaa_sec_job *cf;
1424         dpaa_sec_session *ses;
1425         struct dpaa_sec_op_ctx *ctx;
1426         uint32_t auth_only_len;
1427         struct qman_fq *inq[DPAA_SEC_BURST];
1428
1429         while (nb_ops) {
1430                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1431                                 DPAA_SEC_BURST : nb_ops;
1432                 for (loop = 0; loop < frames_to_send; loop++) {
1433                         op = *(ops++);
1434                         switch (op->sess_type) {
1435                         case RTE_CRYPTO_OP_WITH_SESSION:
1436                                 ses = (dpaa_sec_session *)
1437                                         get_session_private_data(
1438                                                         op->sym->session,
1439                                                         cryptodev_driver_id);
1440                                 break;
1441                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1442                                 ses = (dpaa_sec_session *)
1443                                         get_sec_session_private_data(
1444                                                         op->sym->sec_session);
1445                                 break;
1446                         default:
1447                                 PMD_TX_LOG(ERR,
1448                                         "sessionless crypto op not supported");
1449                                 frames_to_send = loop;
1450                                 nb_ops = loop;
1451                                 goto send_pkts;
1452                         }
1453                         if (unlikely(!ses->qp || ses->qp != qp)) {
1454                                 PMD_INIT_LOG(DEBUG, "sess->qp - %p qp %p",
1455                                                 ses->qp, qp);
1456                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1457                                         frames_to_send = loop;
1458                                         nb_ops = loop;
1459                                         goto send_pkts;
1460                                 }
1461                         }
1462
1463                         auth_only_len = op->sym->auth.data.length -
1464                                                 op->sym->cipher.data.length;
1465                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1466                                 if (is_auth_only(ses)) {
1467                                         cf = build_auth_only(op, ses);
1468                                 } else if (is_cipher_only(ses)) {
1469                                         cf = build_cipher_only(op, ses);
1470                                 } else if (is_aead(ses)) {
1471                                         cf = build_cipher_auth_gcm(op, ses);
1472                                         auth_only_len = ses->auth_only_len;
1473                                 } else if (is_auth_cipher(ses)) {
1474                                         cf = build_cipher_auth(op, ses);
1475                                 } else if (is_proto_ipsec(ses)) {
1476                                         cf = build_proto(op, ses);
1477                                 } else {
1478                                         PMD_TX_LOG(ERR, "not supported sec op");
1479                                         frames_to_send = loop;
1480                                         nb_ops = loop;
1481                                         goto send_pkts;
1482                                 }
1483                         } else {
1484                                 if (is_auth_only(ses)) {
1485                                         cf = build_auth_only_sg(op, ses);
1486                                 } else if (is_cipher_only(ses)) {
1487                                         cf = build_cipher_only_sg(op, ses);
1488                                 } else if (is_aead(ses)) {
1489                                         cf = build_cipher_auth_gcm_sg(op, ses);
1490                                         auth_only_len = ses->auth_only_len;
1491                                 } else if (is_auth_cipher(ses)) {
1492                                         cf = build_cipher_auth_sg(op, ses);
1493                                 } else {
1494                                         PMD_TX_LOG(ERR, "not supported sec op");
1495                                         frames_to_send = loop;
1496                                         nb_ops = loop;
1497                                         goto send_pkts;
1498                                 }
1499                         }
1500                         if (unlikely(!cf)) {
1501                                 frames_to_send = loop;
1502                                 nb_ops = loop;
1503                                 goto send_pkts;
1504                         }
1505
1506                         fd = &fds[loop];
1507                         inq[loop] = ses->inq;
1508                         fd->opaque_addr = 0;
1509                         fd->cmd = 0;
1510                         ctx = container_of(cf, struct dpaa_sec_op_ctx, job);
1511                         qm_fd_addr_set64(fd, dpaa_mem_vtop_ctx(ctx, cf->sg));
1512                         fd->_format1 = qm_fd_compound;
1513                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1514                         /* Auth_only_len is set as 0 in descriptor and it is
1515                          * overwritten here in the fd.cmd which will update
1516                          * the DPOVRD reg.
1517                          */
1518                         if (auth_only_len)
1519                                 fd->cmd = 0x80000000 | auth_only_len;
1520
1521                 }
1522 send_pkts:
1523                 loop = 0;
1524                 while (loop < frames_to_send) {
1525                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1526                                         frames_to_send - loop);
1527                 }
1528                 nb_ops -= frames_to_send;
1529                 num_tx += frames_to_send;
1530         }
1531
1532         dpaa_qp->tx_pkts += num_tx;
1533         dpaa_qp->tx_errs += nb_ops - num_tx;
1534
1535         return num_tx;
1536 }
1537
1538 static uint16_t
1539 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1540                        uint16_t nb_ops)
1541 {
1542         uint16_t num_rx;
1543         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1544
1545         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1546
1547         dpaa_qp->rx_pkts += num_rx;
1548         dpaa_qp->rx_errs += nb_ops - num_rx;
1549
1550         PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx);
1551
1552         return num_rx;
1553 }
1554
1555 /** Release queue pair */
1556 static int
1557 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1558                             uint16_t qp_id)
1559 {
1560         struct dpaa_sec_dev_private *internals;
1561         struct dpaa_sec_qp *qp = NULL;
1562
1563         PMD_INIT_FUNC_TRACE();
1564
1565         PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id);
1566
1567         internals = dev->data->dev_private;
1568         if (qp_id >= internals->max_nb_queue_pairs) {
1569                 PMD_INIT_LOG(ERR, "Max supported qpid %d",
1570                              internals->max_nb_queue_pairs);
1571                 return -EINVAL;
1572         }
1573
1574         qp = &internals->qps[qp_id];
1575         qp->internals = NULL;
1576         dev->data->queue_pairs[qp_id] = NULL;
1577
1578         return 0;
1579 }
1580
1581 /** Setup a queue pair */
1582 static int
1583 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1584                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1585                 __rte_unused int socket_id,
1586                 __rte_unused struct rte_mempool *session_pool)
1587 {
1588         struct dpaa_sec_dev_private *internals;
1589         struct dpaa_sec_qp *qp = NULL;
1590
1591         PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p",
1592                      dev, qp_id, qp_conf);
1593
1594         internals = dev->data->dev_private;
1595         if (qp_id >= internals->max_nb_queue_pairs) {
1596                 PMD_INIT_LOG(ERR, "Max supported qpid %d",
1597                              internals->max_nb_queue_pairs);
1598                 return -EINVAL;
1599         }
1600
1601         qp = &internals->qps[qp_id];
1602         qp->internals = internals;
1603         dev->data->queue_pairs[qp_id] = qp;
1604
1605         return 0;
1606 }
1607
1608 /** Start queue pair */
1609 static int
1610 dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev,
1611                           __rte_unused uint16_t queue_pair_id)
1612 {
1613         PMD_INIT_FUNC_TRACE();
1614
1615         return 0;
1616 }
1617
1618 /** Stop queue pair */
1619 static int
1620 dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev,
1621                          __rte_unused uint16_t queue_pair_id)
1622 {
1623         PMD_INIT_FUNC_TRACE();
1624
1625         return 0;
1626 }
1627
1628 /** Return the number of allocated queue pairs */
1629 static uint32_t
1630 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1631 {
1632         PMD_INIT_FUNC_TRACE();
1633
1634         return dev->data->nb_queue_pairs;
1635 }
1636
1637 /** Returns the size of session structure */
1638 static unsigned int
1639 dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused)
1640 {
1641         PMD_INIT_FUNC_TRACE();
1642
1643         return sizeof(dpaa_sec_session);
1644 }
1645
1646 static int
1647 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1648                      struct rte_crypto_sym_xform *xform,
1649                      dpaa_sec_session *session)
1650 {
1651         session->cipher_alg = xform->cipher.algo;
1652         session->iv.length = xform->cipher.iv.length;
1653         session->iv.offset = xform->cipher.iv.offset;
1654         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1655                                                RTE_CACHE_LINE_SIZE);
1656         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1657                 PMD_INIT_LOG(ERR, "No Memory for cipher key\n");
1658                 return -ENOMEM;
1659         }
1660         session->cipher_key.length = xform->cipher.key.length;
1661
1662         memcpy(session->cipher_key.data, xform->cipher.key.data,
1663                xform->cipher.key.length);
1664         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1665                         DIR_ENC : DIR_DEC;
1666
1667         return 0;
1668 }
1669
1670 static int
1671 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1672                    struct rte_crypto_sym_xform *xform,
1673                    dpaa_sec_session *session)
1674 {
1675         session->auth_alg = xform->auth.algo;
1676         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1677                                              RTE_CACHE_LINE_SIZE);
1678         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1679                 PMD_INIT_LOG(ERR, "No Memory for auth key\n");
1680                 return -ENOMEM;
1681         }
1682         session->auth_key.length = xform->auth.key.length;
1683         session->digest_length = xform->auth.digest_length;
1684
1685         memcpy(session->auth_key.data, xform->auth.key.data,
1686                xform->auth.key.length);
1687         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1688                         DIR_ENC : DIR_DEC;
1689
1690         return 0;
1691 }
1692
1693 static int
1694 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1695                    struct rte_crypto_sym_xform *xform,
1696                    dpaa_sec_session *session)
1697 {
1698         session->aead_alg = xform->aead.algo;
1699         session->iv.length = xform->aead.iv.length;
1700         session->iv.offset = xform->aead.iv.offset;
1701         session->auth_only_len = xform->aead.aad_length;
1702         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1703                                              RTE_CACHE_LINE_SIZE);
1704         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1705                 PMD_INIT_LOG(ERR, "No Memory for aead key\n");
1706                 return -ENOMEM;
1707         }
1708         session->aead_key.length = xform->aead.key.length;
1709         session->digest_length = xform->aead.digest_length;
1710
1711         memcpy(session->aead_key.data, xform->aead.key.data,
1712                xform->aead.key.length);
1713         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1714                         DIR_ENC : DIR_DEC;
1715
1716         return 0;
1717 }
1718
1719 static struct qman_fq *
1720 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1721 {
1722         unsigned int i;
1723
1724         for (i = 0; i < qi->max_nb_sessions; i++) {
1725                 if (qi->inq_attach[i] == 0) {
1726                         qi->inq_attach[i] = 1;
1727                         return &qi->inq[i];
1728                 }
1729         }
1730         PMD_DRV_LOG(ERR, "All ses session in use %x", qi->max_nb_sessions);
1731
1732         return NULL;
1733 }
1734
1735 static int
1736 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1737 {
1738         unsigned int i;
1739
1740         for (i = 0; i < qi->max_nb_sessions; i++) {
1741                 if (&qi->inq[i] == fq) {
1742                         qman_retire_fq(fq, NULL);
1743                         qman_oos_fq(fq);
1744                         qi->inq_attach[i] = 0;
1745                         return 0;
1746                 }
1747         }
1748         return -1;
1749 }
1750
1751 static int
1752 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1753 {
1754         int ret;
1755
1756         sess->qp = qp;
1757         ret = dpaa_sec_prep_cdb(sess);
1758         if (ret) {
1759                 PMD_DRV_LOG(ERR, "Unable to prepare sec cdb");
1760                 return -1;
1761         }
1762
1763         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1764                                qman_fq_fqid(&qp->outq));
1765         if (ret)
1766                 PMD_DRV_LOG(ERR, "Unable to init sec queue");
1767
1768         return ret;
1769 }
1770
1771 static int
1772 dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused,
1773                         uint16_t qp_id __rte_unused,
1774                         void *ses __rte_unused)
1775 {
1776         PMD_INIT_FUNC_TRACE();
1777         return 0;
1778 }
1779
1780 static int
1781 dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev,
1782                         uint16_t qp_id  __rte_unused,
1783                         void *ses)
1784 {
1785         dpaa_sec_session *sess = ses;
1786         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1787
1788         PMD_INIT_FUNC_TRACE();
1789
1790         if (sess->inq)
1791                 dpaa_sec_detach_rxq(qi, sess->inq);
1792         sess->inq = NULL;
1793
1794         sess->qp = NULL;
1795
1796         return 0;
1797 }
1798
1799 static int
1800 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1801                             struct rte_crypto_sym_xform *xform, void *sess)
1802 {
1803         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1804         dpaa_sec_session *session = sess;
1805
1806         PMD_INIT_FUNC_TRACE();
1807
1808         if (unlikely(sess == NULL)) {
1809                 RTE_LOG(ERR, PMD, "invalid session struct\n");
1810                 return -EINVAL;
1811         }
1812
1813         /* Default IV length = 0 */
1814         session->iv.length = 0;
1815
1816         /* Cipher Only */
1817         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1818                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1819                 dpaa_sec_cipher_init(dev, xform, session);
1820
1821         /* Authentication Only */
1822         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1823                    xform->next == NULL) {
1824                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1825                 dpaa_sec_auth_init(dev, xform, session);
1826
1827         /* Cipher then Authenticate */
1828         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1829                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1830                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1831                         dpaa_sec_cipher_init(dev, xform, session);
1832                         dpaa_sec_auth_init(dev, xform->next, session);
1833                 } else {
1834                         PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher");
1835                         return -EINVAL;
1836                 }
1837
1838         /* Authenticate then Cipher */
1839         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1840                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1841                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1842                         dpaa_sec_auth_init(dev, xform, session);
1843                         dpaa_sec_cipher_init(dev, xform->next, session);
1844                 } else {
1845                         PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher");
1846                         return -EINVAL;
1847                 }
1848
1849         /* AEAD operation for AES-GCM kind of Algorithms */
1850         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1851                    xform->next == NULL) {
1852                 dpaa_sec_aead_init(dev, xform, session);
1853
1854         } else {
1855                 PMD_DRV_LOG(ERR, "Invalid crypto type");
1856                 return -EINVAL;
1857         }
1858         session->ctx_pool = internals->ctx_pool;
1859         session->inq = dpaa_sec_attach_rxq(internals);
1860         if (session->inq == NULL) {
1861                 PMD_DRV_LOG(ERR, "unable to attach sec queue");
1862                 goto err1;
1863         }
1864
1865         return 0;
1866
1867 err1:
1868         rte_free(session->cipher_key.data);
1869         rte_free(session->auth_key.data);
1870         memset(session, 0, sizeof(dpaa_sec_session));
1871
1872         return -EINVAL;
1873 }
1874
1875 static int
1876 dpaa_sec_session_configure(struct rte_cryptodev *dev,
1877                 struct rte_crypto_sym_xform *xform,
1878                 struct rte_cryptodev_sym_session *sess,
1879                 struct rte_mempool *mempool)
1880 {
1881         void *sess_private_data;
1882         int ret;
1883
1884         PMD_INIT_FUNC_TRACE();
1885
1886         if (rte_mempool_get(mempool, &sess_private_data)) {
1887                 CDEV_LOG_ERR(
1888                         "Couldn't get object from session mempool");
1889                 return -ENOMEM;
1890         }
1891
1892         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1893         if (ret != 0) {
1894                 PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure "
1895                                 "session parameters");
1896
1897                 /* Return session to mempool */
1898                 rte_mempool_put(mempool, sess_private_data);
1899                 return ret;
1900         }
1901
1902         set_session_private_data(sess, dev->driver_id,
1903                         sess_private_data);
1904
1905
1906         return 0;
1907 }
1908
1909 /** Clear the memory of session so it doesn't leave key material behind */
1910 static void
1911 dpaa_sec_session_clear(struct rte_cryptodev *dev,
1912                 struct rte_cryptodev_sym_session *sess)
1913 {
1914         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1915         uint8_t index = dev->driver_id;
1916         void *sess_priv = get_session_private_data(sess, index);
1917
1918         PMD_INIT_FUNC_TRACE();
1919
1920         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1921
1922         if (sess_priv) {
1923                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1924
1925                 if (s->inq)
1926                         dpaa_sec_detach_rxq(qi, s->inq);
1927                 rte_free(s->cipher_key.data);
1928                 rte_free(s->auth_key.data);
1929                 memset(s, 0, sizeof(dpaa_sec_session));
1930                 set_session_private_data(sess, index, NULL);
1931                 rte_mempool_put(sess_mp, sess_priv);
1932         }
1933 }
1934
1935 static int
1936 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1937                            struct rte_security_session_conf *conf,
1938                            void *sess)
1939 {
1940         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1941         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1942         struct rte_crypto_auth_xform *auth_xform;
1943         struct rte_crypto_cipher_xform *cipher_xform;
1944         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1945
1946         PMD_INIT_FUNC_TRACE();
1947
1948         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1949                 cipher_xform = &conf->crypto_xform->cipher;
1950                 auth_xform = &conf->crypto_xform->next->auth;
1951         } else {
1952                 auth_xform = &conf->crypto_xform->auth;
1953                 cipher_xform = &conf->crypto_xform->next->cipher;
1954         }
1955         session->proto_alg = conf->protocol;
1956         session->cipher_key.data = rte_zmalloc(NULL,
1957                                                cipher_xform->key.length,
1958                                                RTE_CACHE_LINE_SIZE);
1959         if (session->cipher_key.data == NULL &&
1960                         cipher_xform->key.length > 0) {
1961                 RTE_LOG(ERR, PMD, "No Memory for cipher key\n");
1962                 return -ENOMEM;
1963         }
1964
1965         session->cipher_key.length = cipher_xform->key.length;
1966         session->auth_key.data = rte_zmalloc(NULL,
1967                                         auth_xform->key.length,
1968                                         RTE_CACHE_LINE_SIZE);
1969         if (session->auth_key.data == NULL &&
1970                         auth_xform->key.length > 0) {
1971                 RTE_LOG(ERR, PMD, "No Memory for auth key\n");
1972                 rte_free(session->cipher_key.data);
1973                 return -ENOMEM;
1974         }
1975         session->auth_key.length = auth_xform->key.length;
1976         memcpy(session->cipher_key.data, cipher_xform->key.data,
1977                         cipher_xform->key.length);
1978         memcpy(session->auth_key.data, auth_xform->key.data,
1979                         auth_xform->key.length);
1980
1981         switch (auth_xform->algo) {
1982         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1983                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1984                 break;
1985         case RTE_CRYPTO_AUTH_MD5_HMAC:
1986                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1987                 break;
1988         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1989                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1990                 break;
1991         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1992                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1993                 break;
1994         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1995                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1996                 break;
1997         case RTE_CRYPTO_AUTH_AES_CMAC:
1998                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1999                 break;
2000         case RTE_CRYPTO_AUTH_NULL:
2001                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2002                 break;
2003         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2004         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2005         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2006         case RTE_CRYPTO_AUTH_SHA1:
2007         case RTE_CRYPTO_AUTH_SHA256:
2008         case RTE_CRYPTO_AUTH_SHA512:
2009         case RTE_CRYPTO_AUTH_SHA224:
2010         case RTE_CRYPTO_AUTH_SHA384:
2011         case RTE_CRYPTO_AUTH_MD5:
2012         case RTE_CRYPTO_AUTH_AES_GMAC:
2013         case RTE_CRYPTO_AUTH_KASUMI_F9:
2014         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2015         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2016                 RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n",
2017                         auth_xform->algo);
2018                 goto out;
2019         default:
2020                 RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n",
2021                         auth_xform->algo);
2022                 goto out;
2023         }
2024
2025         switch (cipher_xform->algo) {
2026         case RTE_CRYPTO_CIPHER_AES_CBC:
2027                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2028                 break;
2029         case RTE_CRYPTO_CIPHER_3DES_CBC:
2030                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2031                 break;
2032         case RTE_CRYPTO_CIPHER_AES_CTR:
2033                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2034                 break;
2035         case RTE_CRYPTO_CIPHER_NULL:
2036         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2037         case RTE_CRYPTO_CIPHER_3DES_ECB:
2038         case RTE_CRYPTO_CIPHER_AES_ECB:
2039         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2040                 RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n",
2041                         cipher_xform->algo);
2042                 goto out;
2043         default:
2044                 RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n",
2045                         cipher_xform->algo);
2046                 goto out;
2047         }
2048
2049         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2050                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2051                                 sizeof(session->ip4_hdr));
2052                 session->ip4_hdr.ip_v = IPVERSION;
2053                 session->ip4_hdr.ip_hl = 5;
2054                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2055                                                 sizeof(session->ip4_hdr));
2056                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2057                 session->ip4_hdr.ip_id = 0;
2058                 session->ip4_hdr.ip_off = 0;
2059                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2060                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2061                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2062                                 : IPPROTO_AH;
2063                 session->ip4_hdr.ip_sum = 0;
2064                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2065                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2066                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2067                                                 (void *)&session->ip4_hdr,
2068                                                 sizeof(struct ip));
2069
2070                 session->encap_pdb.options =
2071                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2072                         PDBOPTS_ESP_OIHI_PDB_INL |
2073                         PDBOPTS_ESP_IVSRC |
2074                         PDBHMO_ESP_ENCAP_DTTL;
2075                 session->encap_pdb.spi = ipsec_xform->spi;
2076                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2077
2078                 session->dir = DIR_ENC;
2079         } else if (ipsec_xform->direction ==
2080                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2081                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2082                 session->decap_pdb.options = sizeof(struct ip) << 16;
2083                 session->dir = DIR_DEC;
2084         } else
2085                 goto out;
2086         session->ctx_pool = internals->ctx_pool;
2087         session->inq = dpaa_sec_attach_rxq(internals);
2088         if (session->inq == NULL) {
2089                 PMD_DRV_LOG(ERR, "unable to attach sec queue");
2090                 goto out;
2091         }
2092
2093
2094         return 0;
2095 out:
2096         rte_free(session->auth_key.data);
2097         rte_free(session->cipher_key.data);
2098         memset(session, 0, sizeof(dpaa_sec_session));
2099         return -1;
2100 }
2101
2102 static int
2103 dpaa_sec_security_session_create(void *dev,
2104                                  struct rte_security_session_conf *conf,
2105                                  struct rte_security_session *sess,
2106                                  struct rte_mempool *mempool)
2107 {
2108         void *sess_private_data;
2109         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2110         int ret;
2111
2112         if (rte_mempool_get(mempool, &sess_private_data)) {
2113                 CDEV_LOG_ERR(
2114                         "Couldn't get object from session mempool");
2115                 return -ENOMEM;
2116         }
2117
2118         switch (conf->protocol) {
2119         case RTE_SECURITY_PROTOCOL_IPSEC:
2120                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2121                                 sess_private_data);
2122                 break;
2123         case RTE_SECURITY_PROTOCOL_MACSEC:
2124                 return -ENOTSUP;
2125         default:
2126                 return -EINVAL;
2127         }
2128         if (ret != 0) {
2129                 PMD_DRV_LOG(ERR,
2130                         "DPAA2 PMD: failed to configure session parameters");
2131
2132                 /* Return session to mempool */
2133                 rte_mempool_put(mempool, sess_private_data);
2134                 return ret;
2135         }
2136
2137         set_sec_session_private_data(sess, sess_private_data);
2138
2139         return ret;
2140 }
2141
2142 /** Clear the memory of session so it doesn't leave key material behind */
2143 static int
2144 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2145                 struct rte_security_session *sess)
2146 {
2147         PMD_INIT_FUNC_TRACE();
2148         void *sess_priv = get_sec_session_private_data(sess);
2149
2150         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2151
2152         if (sess_priv) {
2153                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2154
2155                 rte_free(s->cipher_key.data);
2156                 rte_free(s->auth_key.data);
2157                 memset(sess, 0, sizeof(dpaa_sec_session));
2158                 set_sec_session_private_data(sess, NULL);
2159                 rte_mempool_put(sess_mp, sess_priv);
2160         }
2161         return 0;
2162 }
2163
2164
2165 static int
2166 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
2167                        struct rte_cryptodev_config *config __rte_unused)
2168 {
2169         PMD_INIT_FUNC_TRACE();
2170
2171         return 0;
2172 }
2173
2174 static int
2175 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2176 {
2177         PMD_INIT_FUNC_TRACE();
2178         return 0;
2179 }
2180
2181 static void
2182 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2183 {
2184         PMD_INIT_FUNC_TRACE();
2185 }
2186
2187 static int
2188 dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused)
2189 {
2190         PMD_INIT_FUNC_TRACE();
2191         return 0;
2192 }
2193
2194 static void
2195 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2196                        struct rte_cryptodev_info *info)
2197 {
2198         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2199
2200         PMD_INIT_FUNC_TRACE();
2201         if (info != NULL) {
2202                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2203                 info->feature_flags = dev->feature_flags;
2204                 info->capabilities = dpaa_sec_capabilities;
2205                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2206                 info->sym.max_nb_sessions_per_qp =
2207                         RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS /
2208                         RTE_DPAA_MAX_NB_SEC_QPS;
2209                 info->driver_id = cryptodev_driver_id;
2210         }
2211 }
2212
2213 static struct rte_cryptodev_ops crypto_ops = {
2214         .dev_configure        = dpaa_sec_dev_configure,
2215         .dev_start            = dpaa_sec_dev_start,
2216         .dev_stop             = dpaa_sec_dev_stop,
2217         .dev_close            = dpaa_sec_dev_close,
2218         .dev_infos_get        = dpaa_sec_dev_infos_get,
2219         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2220         .queue_pair_release   = dpaa_sec_queue_pair_release,
2221         .queue_pair_start     = dpaa_sec_queue_pair_start,
2222         .queue_pair_stop      = dpaa_sec_queue_pair_stop,
2223         .queue_pair_count     = dpaa_sec_queue_pair_count,
2224         .session_get_size     = dpaa_sec_session_get_size,
2225         .session_configure    = dpaa_sec_session_configure,
2226         .session_clear        = dpaa_sec_session_clear,
2227         .qp_attach_session    = dpaa_sec_qp_attach_sess,
2228         .qp_detach_session    = dpaa_sec_qp_detach_sess,
2229 };
2230
2231 static const struct rte_security_capability *
2232 dpaa_sec_capabilities_get(void *device __rte_unused)
2233 {
2234         return dpaa_sec_security_cap;
2235 }
2236
2237 struct rte_security_ops dpaa_sec_security_ops = {
2238         .session_create = dpaa_sec_security_session_create,
2239         .session_update = NULL,
2240         .session_stats_get = NULL,
2241         .session_destroy = dpaa_sec_security_session_destroy,
2242         .set_pkt_metadata = NULL,
2243         .capabilities_get = dpaa_sec_capabilities_get
2244 };
2245
2246 static int
2247 dpaa_sec_uninit(struct rte_cryptodev *dev)
2248 {
2249         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2250
2251         if (dev == NULL)
2252                 return -ENODEV;
2253
2254         rte_free(dev->security_ctx);
2255
2256         rte_mempool_free(internals->ctx_pool);
2257         rte_free(internals);
2258
2259         PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n",
2260                      dev->data->name, rte_socket_id());
2261
2262         return 0;
2263 }
2264
2265 static int
2266 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2267 {
2268         struct dpaa_sec_dev_private *internals;
2269         struct rte_security_ctx *security_instance;
2270         struct dpaa_sec_qp *qp;
2271         uint32_t i, flags;
2272         int ret;
2273         char str[20];
2274
2275         PMD_INIT_FUNC_TRACE();
2276
2277         cryptodev->driver_id = cryptodev_driver_id;
2278         cryptodev->dev_ops = &crypto_ops;
2279
2280         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2281         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2282         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2283                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2284                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2285                         RTE_CRYPTODEV_FF_SECURITY |
2286                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
2287
2288         internals = cryptodev->data->dev_private;
2289         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2290         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2291
2292         /*
2293          * For secondary processes, we don't initialise any further as primary
2294          * has already done this work. Only check we don't need a different
2295          * RX function
2296          */
2297         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2298                 PMD_INIT_LOG(DEBUG, "Device already init by primary process");
2299                 return 0;
2300         }
2301
2302         /* Initialize security_ctx only for primary process*/
2303         security_instance = rte_malloc("rte_security_instances_ops",
2304                                 sizeof(struct rte_security_ctx), 0);
2305         if (security_instance == NULL)
2306                 return -ENOMEM;
2307         security_instance->device = (void *)cryptodev;
2308         security_instance->ops = &dpaa_sec_security_ops;
2309         security_instance->sess_cnt = 0;
2310         cryptodev->security_ctx = security_instance;
2311
2312         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2313                 /* init qman fq for queue pair */
2314                 qp = &internals->qps[i];
2315                 ret = dpaa_sec_init_tx(&qp->outq);
2316                 if (ret) {
2317                         PMD_INIT_LOG(ERR, "config tx of queue pair  %d", i);
2318                         goto init_error;
2319                 }
2320         }
2321
2322         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2323                 QMAN_FQ_FLAG_TO_DCPORTAL;
2324         for (i = 0; i < internals->max_nb_sessions; i++) {
2325                 /* create rx qman fq for sessions*/
2326                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2327                 if (unlikely(ret != 0)) {
2328                         PMD_INIT_LOG(ERR, "sec qman_create_fq failed");
2329                         goto init_error;
2330                 }
2331         }
2332
2333         sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id);
2334         internals->ctx_pool = rte_mempool_create((const char *)str,
2335                         CTX_POOL_NUM_BUFS,
2336                         CTX_POOL_BUF_SIZE,
2337                         CTX_POOL_CACHE_SIZE, 0,
2338                         NULL, NULL, NULL, NULL,
2339                         SOCKET_ID_ANY, 0);
2340         if (!internals->ctx_pool) {
2341                 RTE_LOG(ERR, PMD, "%s create failed\n", str);
2342                 goto init_error;
2343         }
2344
2345         PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name);
2346         return 0;
2347
2348 init_error:
2349         PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name);
2350
2351         dpaa_sec_uninit(cryptodev);
2352         return -EFAULT;
2353 }
2354
2355 static int
2356 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv,
2357                                 struct rte_dpaa_device *dpaa_dev)
2358 {
2359         struct rte_cryptodev *cryptodev;
2360         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2361
2362         int retval;
2363
2364         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2365
2366         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2367         if (cryptodev == NULL)
2368                 return -ENOMEM;
2369
2370         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2371                 cryptodev->data->dev_private = rte_zmalloc_socket(
2372                                         "cryptodev private structure",
2373                                         sizeof(struct dpaa_sec_dev_private),
2374                                         RTE_CACHE_LINE_SIZE,
2375                                         rte_socket_id());
2376
2377                 if (cryptodev->data->dev_private == NULL)
2378                         rte_panic("Cannot allocate memzone for private "
2379                                         "device data");
2380         }
2381
2382         dpaa_dev->crypto_dev = cryptodev;
2383         cryptodev->device = &dpaa_dev->device;
2384         cryptodev->device->driver = &dpaa_drv->driver;
2385
2386         /* init user callbacks */
2387         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2388
2389         /* if sec device version is not configured */
2390         if (!rta_get_sec_era()) {
2391                 const struct device_node *caam_node;
2392
2393                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2394                         const uint32_t *prop = of_get_property(caam_node,
2395                                         "fsl,sec-era",
2396                                         NULL);
2397                         if (prop) {
2398                                 rta_set_sec_era(
2399                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2400                                 break;
2401                         }
2402                 }
2403         }
2404
2405         /* Invoke PMD device initialization function */
2406         retval = dpaa_sec_dev_init(cryptodev);
2407         if (retval == 0)
2408                 return 0;
2409
2410         /* In case of error, cleanup is done */
2411         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2412                 rte_free(cryptodev->data->dev_private);
2413
2414         rte_cryptodev_pmd_release_device(cryptodev);
2415
2416         return -ENXIO;
2417 }
2418
2419 static int
2420 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2421 {
2422         struct rte_cryptodev *cryptodev;
2423         int ret;
2424
2425         cryptodev = dpaa_dev->crypto_dev;
2426         if (cryptodev == NULL)
2427                 return -ENODEV;
2428
2429         ret = dpaa_sec_uninit(cryptodev);
2430         if (ret)
2431                 return ret;
2432
2433         return rte_cryptodev_pmd_destroy(cryptodev);
2434 }
2435
2436 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2437         .drv_type = FSL_DPAA_CRYPTO,
2438         .driver = {
2439                 .name = "DPAA SEC PMD"
2440         },
2441         .probe = cryptodev_dpaa_sec_probe,
2442         .remove = cryptodev_dpaa_sec_remove,
2443 };
2444
2445 static struct cryptodev_driver dpaa_sec_crypto_drv;
2446
2447 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2448 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver,
2449                 cryptodev_driver_id);