drivers/crypto/caam_jr/caam_jr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017-2018 NXP
   3  */
   4
   5 #include <fcntl.h>
   6 #include <unistd.h>
   7 #include <sched.h>
   8 #include <net/if.h>
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_common.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_crypto.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_bus_vdev.h>
  16 #include <rte_malloc.h>
  17 #include <rte_security_driver.h>
  18 #include <rte_hexdump.h>
  19
  20 #include <caam_jr_capabilities.h>
  21 #include <caam_jr_config.h>
  22 #include <caam_jr_hw_specific.h>
  23 #include <caam_jr_pvt.h>
  24 #include <caam_jr_desc.h>
  25 #include <caam_jr_log.h>
  26
  27 /* RTA header files */
  28 #include <hw/desc/common.h>
  29 #include <hw/desc/algo.h>
  30 #include <of.h>
  31
  32 #define CAAM_JR_DBG     0
  33 #define CRYPTODEV_NAME_CAAM_JR_PMD      crypto_caam_jr
  34 static uint8_t cryptodev_driver_id;
  35 int caam_jr_logtype;
  36
  37 enum rta_sec_era rta_sec_era;
  38
  39 /* Lists the states possible for the SEC user space driver. */
  40 enum sec_driver_state_e {
  41         SEC_DRIVER_STATE_IDLE,          /* Driver not initialized */
  42         SEC_DRIVER_STATE_STARTED,       /* Driver initialized and can be used*/
  43         SEC_DRIVER_STATE_RELEASE,       /* Driver release is in progress */
  44 };
  45
  46 /* Job rings used for communication with SEC HW */
  47 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
  48
  49 /* The current state of SEC user space driver */
  50 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
  51
  52 /* The number of job rings used by SEC user space driver */
  53 static int g_job_rings_no;
  54 static int g_job_rings_max;
  55
  56 struct sec_outring_entry {
  57         phys_addr_t desc;       /* Pointer to completed descriptor */
  58         uint32_t status;        /* Status for completed descriptor */
  59 } __rte_packed;
  60
  61 /* virtual address conversin when mempool support is available for ctx */
  62 static inline phys_addr_t
  63 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
  64 {
  65         PMD_INIT_FUNC_TRACE();
  66         return (size_t)vaddr - ctx->vtop_offset;
  67 }
  68
  69 static inline void
  70 caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
  71 {
  72         PMD_INIT_FUNC_TRACE();
  73         /* report op status to sym->op and then free the ctx memeory  */
  74         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  75 }
  76
  77 static inline struct caam_jr_op_ctx *
  78 caam_jr_alloc_ctx(struct caam_jr_session *ses)
  79 {
  80         struct caam_jr_op_ctx *ctx;
  81         int ret;
  82
  83         PMD_INIT_FUNC_TRACE();
  84         ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  85         if (!ctx || ret) {
  86                 CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
  87                 return NULL;
  88         }
  89         /*
  90          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  91          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  92          * to clear all the SG entries. caam_jr_alloc_ctx() is called for
  93          * each packet, memset is costlier than dcbz_64().
  94          */
  95         dcbz_64(&ctx->sg[SG_CACHELINE_0]);
  96         dcbz_64(&ctx->sg[SG_CACHELINE_1]);
  97         dcbz_64(&ctx->sg[SG_CACHELINE_2]);
  98         dcbz_64(&ctx->sg[SG_CACHELINE_3]);
  99
 100         ctx->ctx_pool = ses->ctx_pool;
 101         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
 102
 103         return ctx;
 104 }
 105
 106 static
 107 void caam_jr_stats_get(struct rte_cryptodev *dev,
 108                         struct rte_cryptodev_stats *stats)
 109 {
 110         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 111                                         dev->data->queue_pairs;
 112         int i;
 113
 114         PMD_INIT_FUNC_TRACE();
 115         if (stats == NULL) {
 116                 CAAM_JR_ERR("Invalid stats ptr NULL");
 117                 return;
 118         }
 119         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 120                 if (qp[i] == NULL) {
 121                         CAAM_JR_WARN("Uninitialised queue pair");
 122                         continue;
 123                 }
 124
 125                 stats->enqueued_count += qp[i]->tx_pkts;
 126                 stats->dequeued_count += qp[i]->rx_pkts;
 127                 stats->enqueue_err_count += qp[i]->tx_errs;
 128                 stats->dequeue_err_count += qp[i]->rx_errs;
 129                 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
 130                              "\n\tTX Ring Full = %" PRIu64,
 131                              qp[i]->rx_poll_err,
 132                              qp[i]->tx_ring_full);
 133         }
 134 }
 135
 136 static
 137 void caam_jr_stats_reset(struct rte_cryptodev *dev)
 138 {
 139         int i;
 140         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 141                                    (dev->data->queue_pairs);
 142
 143         PMD_INIT_FUNC_TRACE();
 144         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 145                 if (qp[i] == NULL) {
 146                         CAAM_JR_WARN("Uninitialised queue pair");
 147                         continue;
 148                 }
 149                 qp[i]->rx_pkts = 0;
 150                 qp[i]->rx_errs = 0;
 151                 qp[i]->rx_poll_err = 0;
 152                 qp[i]->tx_pkts = 0;
 153                 qp[i]->tx_errs = 0;
 154                 qp[i]->tx_ring_full = 0;
 155         }
 156 }
 157
 158 static inline int
 159 is_cipher_only(struct caam_jr_session *ses)
 160 {
 161         PMD_INIT_FUNC_TRACE();
 162         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 163                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 164 }
 165
 166 static inline int
 167 is_auth_only(struct caam_jr_session *ses)
 168 {
 169         PMD_INIT_FUNC_TRACE();
 170         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 171                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 172 }
 173
 174 static inline int
 175 is_aead(struct caam_jr_session *ses)
 176 {
 177         PMD_INIT_FUNC_TRACE();
 178         return ((ses->cipher_alg == 0) &&
 179                 (ses->auth_alg == 0) &&
 180                 (ses->aead_alg != 0));
 181 }
 182
 183 static inline int
 184 is_auth_cipher(struct caam_jr_session *ses)
 185 {
 186         PMD_INIT_FUNC_TRACE();
 187         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 188                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 189                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 190 }
 191
 192 static inline int
 193 is_proto_ipsec(struct caam_jr_session *ses)
 194 {
 195         PMD_INIT_FUNC_TRACE();
 196         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 197 }
 198
 199 static inline int
 200 is_encode(struct caam_jr_session *ses)
 201 {
 202         PMD_INIT_FUNC_TRACE();
 203         return ses->dir == DIR_ENC;
 204 }
 205
 206 static inline int
 207 is_decode(struct caam_jr_session *ses)
 208 {
 209         PMD_INIT_FUNC_TRACE();
 210         return ses->dir == DIR_DEC;
 211 }
 212
 213 static inline void
 214 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
 215 {
 216         PMD_INIT_FUNC_TRACE();
 217         switch (ses->auth_alg) {
 218         case RTE_CRYPTO_AUTH_NULL:
 219                 ses->digest_length = 0;
 220                 break;
 221         case RTE_CRYPTO_AUTH_MD5_HMAC:
 222                 alginfo_a->algtype =
 223                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 224                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 225                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 226                 break;
 227         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 228                 alginfo_a->algtype =
 229                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 230                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 231                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 232                 break;
 233         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 234                 alginfo_a->algtype =
 235                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 236                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 237                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 238                 break;
 239         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 240                 alginfo_a->algtype =
 241                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 242                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 243                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 244                 break;
 245         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 246                 alginfo_a->algtype =
 247                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 248                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 249                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 250                 break;
 251         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 252                 alginfo_a->algtype =
 253                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 254                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 255                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 256                 break;
 257         default:
 258                 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
 259         }
 260 }
 261
 262 static inline void
 263 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
 264 {
 265         PMD_INIT_FUNC_TRACE();
 266         switch (ses->cipher_alg) {
 267         case RTE_CRYPTO_CIPHER_NULL:
 268                 break;
 269         case RTE_CRYPTO_CIPHER_AES_CBC:
 270                 alginfo_c->algtype =
 271                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 272                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 273                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 274                 break;
 275         case RTE_CRYPTO_CIPHER_3DES_CBC:
 276                 alginfo_c->algtype =
 277                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 278                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 279                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 280                 break;
 281         case RTE_CRYPTO_CIPHER_AES_CTR:
 282                 alginfo_c->algtype =
 283                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 284                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 285                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 286                 break;
 287         default:
 288                 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
 289         }
 290 }
 291
 292 static inline void
 293 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
 294 {
 295         PMD_INIT_FUNC_TRACE();
 296         switch (ses->aead_alg) {
 297         case RTE_CRYPTO_AEAD_AES_GCM:
 298                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 299                 alginfo->algmode = OP_ALG_AAI_GCM;
 300                 break;
 301         default:
 302                 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
 303         }
 304 }
 305
 306 /* prepare command block of the session */
 307 static int
 308 caam_jr_prep_cdb(struct caam_jr_session *ses)
 309 {
 310         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 311         int32_t shared_desc_len = 0;
 312         struct sec_cdb *cdb;
 313         int err;
 314 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 315         int swap = false;
 316 #else
 317         int swap = true;
 318 #endif
 319
 320         PMD_INIT_FUNC_TRACE();
 321         if (ses->cdb)
 322                 caam_jr_dma_free(ses->cdb);
 323
 324         cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
 325         if (!cdb) {
 326                 CAAM_JR_ERR("failed to allocate memory for cdb\n");
 327                 return -1;
 328         }
 329
 330         ses->cdb = cdb;
 331
 332         memset(cdb, 0, sizeof(struct sec_cdb));
 333
 334         if (is_cipher_only(ses)) {
 335                 caam_cipher_alg(ses, &alginfo_c);
 336                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 337                         CAAM_JR_ERR("not supported cipher alg");
 338                         rte_free(cdb);
 339                         return -ENOTSUP;
 340                 }
 341
 342                 alginfo_c.key = (size_t)ses->cipher_key.data;
 343                 alginfo_c.keylen = ses->cipher_key.length;
 344                 alginfo_c.key_enc_flags = 0;
 345                 alginfo_c.key_type = RTA_DATA_IMM;
 346
 347                 shared_desc_len = cnstr_shdsc_blkcipher(
 348                                                 cdb->sh_desc, true,
 349                                                 swap, &alginfo_c,
 350                                                 NULL,
 351                                                 ses->iv.length,
 352                                                 ses->dir);
 353         } else if (is_auth_only(ses)) {
 354                 caam_auth_alg(ses, &alginfo_a);
 355                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 356                         CAAM_JR_ERR("not supported auth alg");
 357                         rte_free(cdb);
 358                         return -ENOTSUP;
 359                 }
 360
 361                 alginfo_a.key = (size_t)ses->auth_key.data;
 362                 alginfo_a.keylen = ses->auth_key.length;
 363                 alginfo_a.key_enc_flags = 0;
 364                 alginfo_a.key_type = RTA_DATA_IMM;
 365
 366                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 367                                                    swap, &alginfo_a,
 368                                                    !ses->dir,
 369                                                    ses->digest_length);
 370         } else if (is_aead(ses)) {
 371                 caam_aead_alg(ses, &alginfo);
 372                 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 373                         CAAM_JR_ERR("not supported aead alg");
 374                         rte_free(cdb);
 375                         return -ENOTSUP;
 376                 }
 377                 alginfo.key = (size_t)ses->aead_key.data;
 378                 alginfo.keylen = ses->aead_key.length;
 379                 alginfo.key_enc_flags = 0;
 380                 alginfo.key_type = RTA_DATA_IMM;
 381
 382                 if (ses->dir == DIR_ENC)
 383                         shared_desc_len = cnstr_shdsc_gcm_encap(
 384                                         cdb->sh_desc, true, swap,
 385                                         &alginfo,
 386                                         ses->iv.length,
 387                                         ses->digest_length);
 388                 else
 389                         shared_desc_len = cnstr_shdsc_gcm_decap(
 390                                         cdb->sh_desc, true, swap,
 391                                         &alginfo,
 392                                         ses->iv.length,
 393                                         ses->digest_length);
 394         } else {
 395                 caam_cipher_alg(ses, &alginfo_c);
 396                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 397                         CAAM_JR_ERR("not supported cipher alg");
 398                         rte_free(cdb);
 399                         return -ENOTSUP;
 400                 }
 401
 402                 alginfo_c.key = (size_t)ses->cipher_key.data;
 403                 alginfo_c.keylen = ses->cipher_key.length;
 404                 alginfo_c.key_enc_flags = 0;
 405                 alginfo_c.key_type = RTA_DATA_IMM;
 406
 407                 caam_auth_alg(ses, &alginfo_a);
 408                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 409                         CAAM_JR_ERR("not supported auth alg");
 410                         rte_free(cdb);
 411                         return -ENOTSUP;
 412                 }
 413
 414                 alginfo_a.key = (size_t)ses->auth_key.data;
 415                 alginfo_a.keylen = ses->auth_key.length;
 416                 alginfo_a.key_enc_flags = 0;
 417                 alginfo_a.key_type = RTA_DATA_IMM;
 418
 419                 cdb->sh_desc[0] = alginfo_c.keylen;
 420                 cdb->sh_desc[1] = alginfo_a.keylen;
 421                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 422                                        MIN_JOB_DESC_SIZE,
 423                                        (unsigned int *)cdb->sh_desc,
 424                                        &cdb->sh_desc[2], 2);
 425
 426                 if (err < 0) {
 427                         CAAM_JR_ERR("Crypto: Incorrect key lengths");
 428                         rte_free(cdb);
 429                         return err;
 430                 }
 431                 if (cdb->sh_desc[2] & 1)
 432                         alginfo_c.key_type = RTA_DATA_IMM;
 433                 else {
 434                         alginfo_c.key = (size_t)caam_jr_mem_vtop(
 435                                                 (void *)(size_t)alginfo_c.key);
 436                         alginfo_c.key_type = RTA_DATA_PTR;
 437                 }
 438                 if (cdb->sh_desc[2] & (1<<1))
 439                         alginfo_a.key_type = RTA_DATA_IMM;
 440                 else {
 441                         alginfo_a.key = (size_t)caam_jr_mem_vtop(
 442                                                 (void *)(size_t)alginfo_a.key);
 443                         alginfo_a.key_type = RTA_DATA_PTR;
 444                 }
 445                 cdb->sh_desc[0] = 0;
 446                 cdb->sh_desc[1] = 0;
 447                 cdb->sh_desc[2] = 0;
 448                 if (is_proto_ipsec(ses)) {
 449                         if (ses->dir == DIR_ENC) {
 450                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 451                                                 cdb->sh_desc,
 452                                                 true, swap, SHR_SERIAL,
 453                                                 &ses->encap_pdb,
 454                                                 (uint8_t *)&ses->ip4_hdr,
 455                                                 &alginfo_c, &alginfo_a);
 456                         } else if (ses->dir == DIR_DEC) {
 457                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 458                                                 cdb->sh_desc,
 459                                                 true, swap, SHR_SERIAL,
 460                                                 &ses->decap_pdb,
 461                                                 &alginfo_c, &alginfo_a);
 462                         }
 463                 } else {
 464                         /* Auth_only_len is set as 0 here and it will be
 465                          * overwritten in fd for each packet.
 466                          */
 467                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 468                                         true, swap, &alginfo_c, &alginfo_a,
 469                                         ses->iv.length, 0,
 470                                         ses->digest_length, ses->dir);
 471                 }
 472         }
 473
 474         if (shared_desc_len < 0) {
 475                 CAAM_JR_ERR("error in preparing command block");
 476                 return shared_desc_len;
 477         }
 478
 479 #if CAAM_JR_DBG
 480         SEC_DUMP_DESC(cdb->sh_desc);
 481 #endif
 482
 483         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 484
 485         return 0;
 486 }
 487
 488 /* @brief Poll the HW for already processed jobs in the JR
 489  * and silently discard the available jobs or notify them to UA
 490  * with indicated error code.
 491  *
 492  * @param [in,out]  job_ring        The job ring to poll.
 493  * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
 494  *                                  descriptors are to be discarded
 495  *                                  or notified to UA with given error_code.
 496  * @param [out] notified_descs    Number of notified descriptors. Can be NULL
 497  *                                      if do_notify is #FALSE
 498  */
 499 static void
 500 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
 501                   uint32_t do_notify,
 502                   uint32_t *notified_descs)
 503 {
 504         int32_t jobs_no_to_discard = 0;
 505         int32_t discarded_descs_no = 0;
 506
 507         PMD_INIT_FUNC_TRACE();
 508         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
 509                 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
 510
 511         jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
 512
 513         /* Discard all jobs */
 514         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
 515                   job_ring, job_ring->pidx, job_ring->cidx,
 516                   jobs_no_to_discard);
 517
 518         while (jobs_no_to_discard > discarded_descs_no) {
 519                 discarded_descs_no++;
 520                 /* Now increment the consumer index for the current job ring,
 521                  * AFTER saving job in temporary location!
 522                  * Increment the consumer index for the current job ring
 523                  */
 524                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 525                                          SEC_JOB_RING_SIZE);
 526
 527                 hw_remove_entries(job_ring, 1);
 528         }
 529
 530         if (do_notify == true) {
 531                 ASSERT(notified_descs != NULL);
 532                 *notified_descs = discarded_descs_no;
 533         }
 534 }
 535
 536 /* @brief Poll the HW for already processed jobs in the JR
 537  * and notify the available jobs to UA.
 538  *
 539  * @param [in]  job_ring        The job ring to poll.
 540  * @param [in]  limit           The maximum number of jobs to notify.
 541  *                              If set to negative value, all available jobs are
 542  *                              notified.
 543  *
 544  * @retval >=0 for No of jobs notified to UA.
 545  * @retval -1 for error
 546  */
 547 static int
 548 hw_poll_job_ring(struct sec_job_ring_t *job_ring,
 549                  struct rte_crypto_op **ops, int32_t limit,
 550                  struct caam_jr_qp *jr_qp)
 551 {
 552         int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
 553         int32_t number_of_jobs_available = 0;
 554         int32_t notified_descs_no = 0;
 555         uint32_t sec_error_code = 0;
 556         struct job_descriptor *current_desc;
 557         phys_addr_t current_desc_addr;
 558         phys_addr_t *temp_addr;
 559         struct caam_jr_op_ctx *ctx;
 560
 561         PMD_INIT_FUNC_TRACE();
 562         /* TODO check for ops have memory*/
 563         /* check here if any JR error that cannot be written
 564          * in the output status word has occurred
 565          */
 566         if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
 567                 CAAM_JR_INFO("err received");
 568                 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
 569                                         GET_JR_REG(JRINT, job_ring));
 570                 if (unlikely(sec_error_code)) {
 571                         hw_job_ring_error_print(job_ring, sec_error_code);
 572                         return -1;
 573                 }
 574         }
 575         /* compute the number of jobs available in the job ring based on the
 576          * producer and consumer index values.
 577          */
 578         number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
 579         /* Compute the number of notifications that need to be raised to UA
 580          * If limit > total number of done jobs -> notify all done jobs
 581          * If limit = 0 -> error
 582          * If limit < total number of done jobs -> notify a number
 583          * of done jobs equal with limit
 584          */
 585         jobs_no_to_notify = (limit > number_of_jobs_available) ?
 586                                 number_of_jobs_available : limit;
 587         CAAM_JR_DP_DEBUG(
 588                 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
 589                 job_ring, job_ring->pidx, job_ring->cidx,
 590                 limit, number_of_jobs_available, jobs_no_to_notify);
 591
 592         rte_smp_rmb();
 593
 594         while (jobs_no_to_notify > notified_descs_no) {
 595                 static uint64_t false_alarm;
 596                 static uint64_t real_poll;
 597
 598                 /* Get job status here */
 599                 sec_error_code = job_ring->output_ring[job_ring->cidx].status;
 600                 /* Get completed descriptor */
 601                 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
 602                 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
 603
 604                 real_poll++;
 605                 /* todo check if it is false alarm no desc present */
 606                 if (!current_desc_addr) {
 607                         false_alarm++;
 608                         printf("false alarm %" PRIu64 "real %" PRIu64
 609                                 " sec_err =0x%x cidx Index =0%d\n",
 610                                 false_alarm, real_poll,
 611                                 sec_error_code, job_ring->cidx);
 612                         rte_panic("CAAM JR descriptor NULL");
 613                         return notified_descs_no;
 614                 }
 615                 current_desc = (struct job_descriptor *)
 616                                 caam_jr_dma_ptov(current_desc_addr);
 617                 /* now increment the consumer index for the current job ring,
 618                  * AFTER saving job in temporary location!
 619                  */
 620                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 621                                  SEC_JOB_RING_SIZE);
 622                 /* Signal that the job has been processed and the slot is free*/
 623                 hw_remove_entries(job_ring, 1);
 624                 /*TODO for multiple ops, packets*/
 625                 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
 626                 if (unlikely(sec_error_code)) {
 627                         CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
 628                                 job_ring->cidx, sec_error_code);
 629                         hw_handle_job_ring_error(job_ring, sec_error_code);
 630                         //todo improve with exact errors
 631                         ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 632                         jr_qp->rx_errs++;
 633                 } else {
 634                         ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 635 #if CAAM_JR_DBG
 636                         if (ctx->op->sym->m_dst) {
 637                                 rte_hexdump(stdout, "PROCESSED",
 638                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
 639                                 rte_pktmbuf_data_len(ctx->op->sym->m_dst));
 640                         } else {
 641                                 rte_hexdump(stdout, "PROCESSED",
 642                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
 643                                 rte_pktmbuf_data_len(ctx->op->sym->m_src));
 644                         }
 645 #endif
 646                 }
 647                 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 648                         struct ip *ip4_hdr;
 649
 650                         if (ctx->op->sym->m_dst) {
 651                                 /*TODO check for ip header or other*/
 652                                 ip4_hdr = (struct ip *)
 653                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
 654                                 ctx->op->sym->m_dst->pkt_len =
 655                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 656                                 ctx->op->sym->m_dst->data_len =
 657                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 658                         } else {
 659                                 ip4_hdr = (struct ip *)
 660                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
 661                                 ctx->op->sym->m_src->pkt_len =
 662                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 663                                 ctx->op->sym->m_src->data_len =
 664                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 665                         }
 666                 }
 667                 *ops = ctx->op;
 668                 caam_jr_op_ending(ctx);
 669                 ops++;
 670                 notified_descs_no++;
 671         }
 672         return notified_descs_no;
 673 }
 674
 675 static uint16_t
 676 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 677                        uint16_t nb_ops)
 678 {
 679         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
 680         struct sec_job_ring_t *ring = jr_qp->ring;
 681         int num_rx;
 682         int ret;
 683
 684         PMD_INIT_FUNC_TRACE();
 685         CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
 686
 687         /* Poll job ring
 688          * If nb_ops < 0 -> poll JR until no more notifications are available.
 689          * If nb_ops > 0 -> poll JR until limit is reached.
 690          */
 691
 692         /* Run hw poll job ring */
 693         num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
 694         if (num_rx < 0) {
 695                 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
 696                 return 0;
 697         }
 698
 699         CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
 700
 701         if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
 702                 if (num_rx < nb_ops) {
 703                         ret = caam_jr_enable_irqs(ring->irq_fd);
 704                         SEC_ASSERT(ret == 0, ret,
 705                         "Failed to enable irqs for job ring %p", ring);
 706                 }
 707         } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
 708
 709                 /* Always enable IRQ generation when in pure IRQ mode */
 710                 ret = caam_jr_enable_irqs(ring->irq_fd);
 711                 SEC_ASSERT(ret == 0, ret,
 712                         "Failed to enable irqs for job ring %p", ring);
 713         }
 714
 715         jr_qp->rx_pkts += num_rx;
 716
 717         return num_rx;
 718 }
 719
 720 /**
 721  * packet looks like:
 722  *              |<----data_len------->|
 723  *    |ip_header|ah_header|icv|payload|
 724  *              ^
 725  *              |
 726  *         mbuf->pkt.data
 727  */
 728 static inline struct caam_jr_op_ctx *
 729 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 730 {
 731         struct rte_crypto_sym_op *sym = op->sym;
 732         struct rte_mbuf *mbuf = sym->m_src;
 733         struct caam_jr_op_ctx *ctx;
 734         struct sec4_sg_entry *sg;
 735         int     length;
 736         struct sec_cdb *cdb;
 737         uint64_t sdesc_offset;
 738         struct sec_job_descriptor_t *jobdescr;
 739         uint8_t extra_segs;
 740
 741         PMD_INIT_FUNC_TRACE();
 742         if (is_decode(ses))
 743                 extra_segs = 2;
 744         else
 745                 extra_segs = 1;
 746
 747         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 748                 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
 749                                 MAX_SG_ENTRIES);
 750                 return NULL;
 751         }
 752
 753         ctx = caam_jr_alloc_ctx(ses);
 754         if (!ctx)
 755                 return NULL;
 756
 757         ctx->op = op;
 758
 759         cdb = ses->cdb;
 760         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 761
 762         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 763
 764         SEC_JD_INIT(jobdescr);
 765         SEC_JD_SET_SD(jobdescr,
 766                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 767                 cdb->sh_hdr.hi.field.idlen);
 768
 769         /* output */
 770         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 771                         0, ses->digest_length);
 772
 773         /*input */
 774         sg = &ctx->sg[0];
 775         length = sym->auth.data.length;
 776         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
 777         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
 778
 779         /* Successive segs */
 780         mbuf = mbuf->next;
 781         while (mbuf) {
 782                 sg++;
 783                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 784                 sg->len = cpu_to_caam32(mbuf->data_len);
 785                 mbuf = mbuf->next;
 786         }
 787
 788         if (is_decode(ses)) {
 789                 /* digest verification case */
 790                 sg++;
 791                 /* hash result or digest, save digest first */
 792                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 793                            ses->digest_length);
 794 #if CAAM_JR_DBG
 795                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 796 #endif
 797                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 798                 sg->len = cpu_to_caam32(ses->digest_length);
 799                 length += ses->digest_length;
 800         } else {
 801                 length -= ses->digest_length;
 802         }
 803
 804         /* last element*/
 805         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 806
 807         SEC_JD_SET_IN_PTR(jobdescr,
 808                 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
 809         /* enabling sg list */
 810         (jobdescr)->seq_in.command.word  |= 0x01000000;
 811
 812         return ctx;
 813 }
 814
 815 static inline struct caam_jr_op_ctx *
 816 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 817 {
 818         struct rte_crypto_sym_op *sym = op->sym;
 819         struct caam_jr_op_ctx *ctx;
 820         struct sec4_sg_entry *sg;
 821         rte_iova_t start_addr;
 822         struct sec_cdb *cdb;
 823         uint64_t sdesc_offset;
 824         struct sec_job_descriptor_t *jobdescr;
 825
 826         PMD_INIT_FUNC_TRACE();
 827         ctx = caam_jr_alloc_ctx(ses);
 828         if (!ctx)
 829                 return NULL;
 830
 831         ctx->op = op;
 832
 833         cdb = ses->cdb;
 834         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 835
 836         start_addr = rte_pktmbuf_iova(sym->m_src);
 837
 838         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 839
 840         SEC_JD_INIT(jobdescr);
 841         SEC_JD_SET_SD(jobdescr,
 842                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 843                 cdb->sh_hdr.hi.field.idlen);
 844
 845         /* output */
 846         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 847                         0, ses->digest_length);
 848
 849         /*input */
 850         if (is_decode(ses)) {
 851                 sg = &ctx->sg[0];
 852                 SEC_JD_SET_IN_PTR(jobdescr,
 853                         (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
 854                         (sym->auth.data.length + ses->digest_length));
 855                 /* enabling sg list */
 856                 (jobdescr)->seq_in.command.word  |= 0x01000000;
 857
 858                 /* hash result or digest, save digest first */
 859                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 860                            ses->digest_length);
 861                 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
 862                 sg->len = cpu_to_caam32(sym->auth.data.length);
 863
 864 #if CAAM_JR_DBG
 865                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 866 #endif
 867                 /* let's check digest by hw */
 868                 sg++;
 869                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 870                 sg->len = cpu_to_caam32(ses->digest_length);
 871                 /* last element*/
 872                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 873         } else {
 874                 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
 875                         sym->auth.data.offset, sym->auth.data.length);
 876         }
 877         return ctx;
 878 }
 879
 880 static inline struct caam_jr_op_ctx *
 881 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 882 {
 883         struct rte_crypto_sym_op *sym = op->sym;
 884         struct rte_mbuf *mbuf = sym->m_src;
 885         struct caam_jr_op_ctx *ctx;
 886         struct sec4_sg_entry *sg, *in_sg;
 887         int length;
 888         struct sec_cdb *cdb;
 889         uint64_t sdesc_offset;
 890         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 891                         ses->iv.offset);
 892         struct sec_job_descriptor_t *jobdescr;
 893         uint8_t reg_segs;
 894
 895         PMD_INIT_FUNC_TRACE();
 896         if (sym->m_dst) {
 897                 mbuf = sym->m_dst;
 898                 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
 899         } else {
 900                 mbuf = sym->m_src;
 901                 reg_segs = mbuf->nb_segs * 2 + 2;
 902         }
 903
 904         if (reg_segs > MAX_SG_ENTRIES) {
 905                 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
 906                                 MAX_SG_ENTRIES);
 907                 return NULL;
 908         }
 909
 910         ctx = caam_jr_alloc_ctx(ses);
 911         if (!ctx)
 912                 return NULL;
 913
 914         ctx->op = op;
 915         cdb = ses->cdb;
 916         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 917
 918         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 919
 920         SEC_JD_INIT(jobdescr);
 921         SEC_JD_SET_SD(jobdescr,
 922                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 923                 cdb->sh_hdr.hi.field.idlen);
 924
 925 #if CAAM_JR_DBG
 926         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
 927                         sym->m_src->data_off, sym->cipher.data.offset,
 928                         sym->cipher.data.length, ses->iv.length);
 929 #endif
 930         /* output */
 931         if (sym->m_dst)
 932                 mbuf = sym->m_dst;
 933         else
 934                 mbuf = sym->m_src;
 935
 936         sg = &ctx->sg[0];
 937         length = sym->cipher.data.length;
 938
 939         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 940                 + sym->cipher.data.offset);
 941         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 942
 943         /* Successive segs */
 944         mbuf = mbuf->next;
 945         while (mbuf) {
 946                 sg++;
 947                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 948                 sg->len = cpu_to_caam32(mbuf->data_len);
 949                 mbuf = mbuf->next;
 950         }
 951         /* last element*/
 952         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 953
 954         SEC_JD_SET_OUT_PTR(jobdescr,
 955                         (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
 956                         length);
 957         /*enabling sg bit */
 958         (jobdescr)->seq_out.command.word  |= 0x01000000;
 959
 960         /*input */
 961         sg++;
 962         mbuf = sym->m_src;
 963         in_sg = sg;
 964
 965         length = sym->cipher.data.length + ses->iv.length;
 966
 967         /* IV */
 968         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
 969         sg->len = cpu_to_caam32(ses->iv.length);
 970
 971         /* 1st seg */
 972         sg++;
 973         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 974                                 + sym->cipher.data.offset);
 975         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 976
 977         /* Successive segs */
 978         mbuf = mbuf->next;
 979         while (mbuf) {
 980                 sg++;
 981                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 982                 sg->len = cpu_to_caam32(mbuf->data_len);
 983                 mbuf = mbuf->next;
 984         }
 985         /* last element*/
 986         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 987
 988
 989         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
 990                                 length);
 991         /*enabling sg bit */
 992         (jobdescr)->seq_in.command.word  |= 0x01000000;
 993
 994         return ctx;
 995 }
 996
 997 static inline struct caam_jr_op_ctx *
 998 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 999 {
1000         struct rte_crypto_sym_op *sym = op->sym;
1001         struct caam_jr_op_ctx *ctx;
1002         struct sec4_sg_entry *sg;
1003         rte_iova_t src_start_addr, dst_start_addr;
1004         struct sec_cdb *cdb;
1005         uint64_t sdesc_offset;
1006         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1007                         ses->iv.offset);
1008         struct sec_job_descriptor_t *jobdescr;
1009
1010         PMD_INIT_FUNC_TRACE();
1011         ctx = caam_jr_alloc_ctx(ses);
1012         if (!ctx)
1013                 return NULL;
1014
1015         ctx->op = op;
1016         cdb = ses->cdb;
1017         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1018
1019         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1020         if (sym->m_dst)
1021                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1022         else
1023                 dst_start_addr = src_start_addr;
1024
1025         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1026
1027         SEC_JD_INIT(jobdescr);
1028         SEC_JD_SET_SD(jobdescr,
1029                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1030                 cdb->sh_hdr.hi.field.idlen);
1031
1032 #if CAAM_JR_DBG
1033         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
1034                         sym->m_src->data_off, sym->cipher.data.offset,
1035                         sym->cipher.data.length, ses->iv.length);
1036 #endif
1037         /* output */
1038         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
1039                         sym->cipher.data.offset,
1040                         sym->cipher.data.length + ses->iv.length);
1041
1042         /*input */
1043         sg = &ctx->sg[0];
1044         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
1045                                 sym->cipher.data.length + ses->iv.length);
1046         /*enabling sg bit */
1047         (jobdescr)->seq_in.command.word  |= 0x01000000;
1048
1049         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1050         sg->len = cpu_to_caam32(ses->iv.length);
1051
1052         sg = &ctx->sg[1];
1053         sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
1054         sg->len = cpu_to_caam32(sym->cipher.data.length);
1055         /* last element*/
1056         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1057
1058         return ctx;
1059 }
1060
1061 /* For decapsulation:
1062  *     Input:
1063  * +----+----------------+--------------------------------+-----+
1064  * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
1065  * +----+----------------+--------------------------------+-----+
1066  *     Output:
1067  * +----+--------------------------+
1068  * | Decrypted & authenticated data |
1069  * +----+--------------------------+
1070  */
1071
1072 static inline struct caam_jr_op_ctx *
1073 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
1074 {
1075         struct rte_crypto_sym_op *sym = op->sym;
1076         struct caam_jr_op_ctx *ctx;
1077         struct sec4_sg_entry *sg, *out_sg, *in_sg;
1078         struct rte_mbuf *mbuf;
1079         uint32_t length = 0;
1080         struct sec_cdb *cdb;
1081         uint64_t sdesc_offset;
1082         uint8_t req_segs;
1083         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1084                         ses->iv.offset);
1085         struct sec_job_descriptor_t *jobdescr;
1086         uint32_t auth_only_len;
1087
1088         PMD_INIT_FUNC_TRACE();
1089         auth_only_len = op->sym->auth.data.length -
1090                                 op->sym->cipher.data.length;
1091
1092         if (sym->m_dst) {
1093                 mbuf = sym->m_dst;
1094                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1095         } else {
1096                 mbuf = sym->m_src;
1097                 req_segs = mbuf->nb_segs * 2 + 3;
1098         }
1099
1100         if (req_segs > MAX_SG_ENTRIES) {
1101                 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1102                                 MAX_SG_ENTRIES);
1103                 return NULL;
1104         }
1105
1106         ctx = caam_jr_alloc_ctx(ses);
1107         if (!ctx)
1108                 return NULL;
1109
1110         ctx->op = op;
1111         cdb = ses->cdb;
1112         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1113
1114         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1115
1116         SEC_JD_INIT(jobdescr);
1117         SEC_JD_SET_SD(jobdescr,
1118                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1119                 cdb->sh_hdr.hi.field.idlen);
1120
1121         /* output */
1122         if (sym->m_dst)
1123                 mbuf = sym->m_dst;
1124         else
1125                 mbuf = sym->m_src;
1126
1127         out_sg = &ctx->sg[0];
1128         if (is_encode(ses))
1129                 length = sym->auth.data.length + ses->digest_length;
1130         else
1131                 length = sym->auth.data.length;
1132
1133         sg = &ctx->sg[0];
1134
1135         /* 1st seg */
1136         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1137                 + sym->auth.data.offset);
1138         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1139
1140         /* Successive segs */
1141         mbuf = mbuf->next;
1142         while (mbuf) {
1143                 sg++;
1144                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1145                 sg->len = cpu_to_caam32(mbuf->data_len);
1146                 mbuf = mbuf->next;
1147         }
1148
1149         if (is_encode(ses)) {
1150                 /* set auth output */
1151                 sg++;
1152                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1153                 sg->len = cpu_to_caam32(ses->digest_length);
1154         }
1155         /* last element*/
1156         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1157
1158         SEC_JD_SET_OUT_PTR(jobdescr,
1159                            (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
1160         /* set sg bit */
1161         (jobdescr)->seq_out.command.word  |= 0x01000000;
1162
1163         /* input */
1164         sg++;
1165         mbuf = sym->m_src;
1166         in_sg = sg;
1167         if (is_encode(ses))
1168                 length = ses->iv.length + sym->auth.data.length;
1169         else
1170                 length = ses->iv.length + sym->auth.data.length
1171                                                 + ses->digest_length;
1172
1173         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1174         sg->len = cpu_to_caam32(ses->iv.length);
1175
1176         sg++;
1177         /* 1st seg */
1178         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1179                 + sym->auth.data.offset);
1180         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1181
1182         /* Successive segs */
1183         mbuf = mbuf->next;
1184         while (mbuf) {
1185                 sg++;
1186                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1187                 sg->len = cpu_to_caam32(mbuf->data_len);
1188                 mbuf = mbuf->next;
1189         }
1190
1191         if (is_decode(ses)) {
1192                 sg++;
1193                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1194                        ses->digest_length);
1195                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1196                 sg->len = cpu_to_caam32(ses->digest_length);
1197         }
1198         /* last element*/
1199         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1200
1201         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
1202                                 length);
1203         /* set sg bit */
1204         (jobdescr)->seq_in.command.word  |= 0x01000000;
1205         /* Auth_only_len is set as 0 in descriptor and it is
1206          * overwritten here in the jd which will update
1207          * the DPOVRD reg.
1208          */
1209         if (auth_only_len)
1210                 /* set sg bit */
1211                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1212
1213         return ctx;
1214 }
1215
1216 static inline struct caam_jr_op_ctx *
1217 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
1218 {
1219         struct rte_crypto_sym_op *sym = op->sym;
1220         struct caam_jr_op_ctx *ctx;
1221         struct sec4_sg_entry *sg;
1222         rte_iova_t src_start_addr, dst_start_addr;
1223         uint32_t length = 0;
1224         struct sec_cdb *cdb;
1225         uint64_t sdesc_offset;
1226         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1227                         ses->iv.offset);
1228         struct sec_job_descriptor_t *jobdescr;
1229         uint32_t auth_only_len;
1230
1231         PMD_INIT_FUNC_TRACE();
1232         auth_only_len = op->sym->auth.data.length -
1233                                 op->sym->cipher.data.length;
1234
1235         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1236         if (sym->m_dst)
1237                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1238         else
1239                 dst_start_addr = src_start_addr;
1240
1241         ctx = caam_jr_alloc_ctx(ses);
1242         if (!ctx)
1243                 return NULL;
1244
1245         ctx->op = op;
1246         cdb = ses->cdb;
1247         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1248
1249         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1250
1251         SEC_JD_INIT(jobdescr);
1252         SEC_JD_SET_SD(jobdescr,
1253                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1254                 cdb->sh_hdr.hi.field.idlen);
1255
1256         /* input */
1257         sg = &ctx->sg[0];
1258         if (is_encode(ses)) {
1259                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1260                 sg->len = cpu_to_caam32(ses->iv.length);
1261                 length += ses->iv.length;
1262
1263                 sg++;
1264                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1265                 sg->len = cpu_to_caam32(sym->auth.data.length);
1266                 length += sym->auth.data.length;
1267                 /* last element*/
1268                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1269         } else {
1270                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1271                 sg->len = cpu_to_caam32(ses->iv.length);
1272                 length += ses->iv.length;
1273
1274                 sg++;
1275                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1276                 sg->len = cpu_to_caam32(sym->auth.data.length);
1277                 length += sym->auth.data.length;
1278
1279                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1280                        ses->digest_length);
1281                 sg++;
1282                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1283                 sg->len = cpu_to_caam32(ses->digest_length);
1284                 length += ses->digest_length;
1285                 /* last element*/
1286                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1287         }
1288
1289         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
1290                                 length);
1291         /* set sg bit */
1292         (jobdescr)->seq_in.command.word  |= 0x01000000;
1293
1294         /* output */
1295         sg = &ctx->sg[6];
1296
1297         sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
1298         sg->len = cpu_to_caam32(sym->cipher.data.length);
1299         length = sym->cipher.data.length;
1300
1301         if (is_encode(ses)) {
1302                 /* set auth output */
1303                 sg++;
1304                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1305                 sg->len = cpu_to_caam32(ses->digest_length);
1306                 length += ses->digest_length;
1307         }
1308         /* last element*/
1309         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1310
1311         SEC_JD_SET_OUT_PTR(jobdescr,
1312                            (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
1313         /* set sg bit */
1314         (jobdescr)->seq_out.command.word  |= 0x01000000;
1315
1316         /* Auth_only_len is set as 0 in descriptor and it is
1317          * overwritten here in the jd which will update
1318          * the DPOVRD reg.
1319          */
1320         if (auth_only_len)
1321                 /* set sg bit */
1322                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1323
1324         return ctx;
1325 }
1326
1327 static inline struct caam_jr_op_ctx *
1328 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
1329 {
1330         struct rte_crypto_sym_op *sym = op->sym;
1331         struct caam_jr_op_ctx *ctx = NULL;
1332         phys_addr_t src_start_addr, dst_start_addr;
1333         struct sec_cdb *cdb;
1334         uint64_t sdesc_offset;
1335         struct sec_job_descriptor_t *jobdescr;
1336
1337         PMD_INIT_FUNC_TRACE();
1338         ctx = caam_jr_alloc_ctx(ses);
1339         if (!ctx)
1340                 return NULL;
1341         ctx->op = op;
1342
1343         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1344         if (sym->m_dst)
1345                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1346         else
1347                 dst_start_addr = src_start_addr;
1348
1349         cdb = ses->cdb;
1350         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1351
1352         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1353
1354         SEC_JD_INIT(jobdescr);
1355         SEC_JD_SET_SD(jobdescr,
1356                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1357                         cdb->sh_hdr.hi.field.idlen);
1358
1359         /* output */
1360         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
1361                         sym->m_src->buf_len - sym->m_src->data_off);
1362         /* input */
1363         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
1364                         sym->m_src->pkt_len);
1365         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1366
1367         return ctx;
1368 }
1369
1370 static int
1371 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
1372 {
1373         struct sec_job_ring_t *ring = qp->ring;
1374         struct caam_jr_session *ses;
1375         struct caam_jr_op_ctx *ctx = NULL;
1376         struct sec_job_descriptor_t *jobdescr __rte_unused;
1377
1378         PMD_INIT_FUNC_TRACE();
1379         switch (op->sess_type) {
1380         case RTE_CRYPTO_OP_WITH_SESSION:
1381                 ses = (struct caam_jr_session *)
1382                 get_sym_session_private_data(op->sym->session,
1383                                         cryptodev_driver_id);
1384                 break;
1385         case RTE_CRYPTO_OP_SECURITY_SESSION:
1386                 ses = (struct caam_jr_session *)
1387                         get_sec_session_private_data(
1388                                         op->sym->sec_session);
1389                 break;
1390         default:
1391                 CAAM_JR_DP_ERR("sessionless crypto op not supported");
1392                 qp->tx_errs++;
1393                 return -1;
1394         }
1395
1396         if (unlikely(!ses->qp || ses->qp != qp)) {
1397                 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
1398                 ses->qp = qp;
1399                 caam_jr_prep_cdb(ses);
1400         }
1401
1402         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1403                 if (is_auth_cipher(ses))
1404                         ctx = build_cipher_auth(op, ses);
1405                 else if (is_aead(ses))
1406                         goto err1;
1407                 else if (is_auth_only(ses))
1408                         ctx = build_auth_only(op, ses);
1409                 else if (is_cipher_only(ses))
1410                         ctx = build_cipher_only(op, ses);
1411                 else if (is_proto_ipsec(ses))
1412                         ctx = build_proto(op, ses);
1413         } else {
1414                 if (is_auth_cipher(ses))
1415                         ctx = build_cipher_auth_sg(op, ses);
1416                 else if (is_aead(ses))
1417                         goto err1;
1418                 else if (is_auth_only(ses))
1419                         ctx = build_auth_only_sg(op, ses);
1420                 else if (is_cipher_only(ses))
1421                         ctx = build_cipher_only_sg(op, ses);
1422         }
1423 err1:
1424         if (unlikely(!ctx)) {
1425                 qp->tx_errs++;
1426                 CAAM_JR_ERR("not supported sec op");
1427                 return -1;
1428         }
1429 #if CAAM_JR_DBG
1430         if (is_decode(ses))
1431                 rte_hexdump(stdout, "DECODE",
1432                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1433                         rte_pktmbuf_data_len(op->sym->m_src));
1434         else
1435                 rte_hexdump(stdout, "ENCODE",
1436                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1437                         rte_pktmbuf_data_len(op->sym->m_src));
1438
1439         printf("\n JD before conversion\n");
1440         for (int i = 0; i < 12; i++)
1441                 printf("\n 0x%08x", ctx->jobdes.desc[i]);
1442 #endif
1443
1444         CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
1445                       ring, ring->pidx, ring->cidx);
1446
1447         /* todo - do we want to retry */
1448         if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
1449                          SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
1450                 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
1451                               ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
1452                 caam_jr_op_ending(ctx);
1453                 qp->tx_ring_full++;
1454                 return -EBUSY;
1455         }
1456
1457 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
1458         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1459
1460         jobdescr->deschdr.command.word =
1461                 cpu_to_caam32(jobdescr->deschdr.command.word);
1462         jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
1463         jobdescr->seq_out.command.word =
1464                 cpu_to_caam32(jobdescr->seq_out.command.word);
1465         jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
1466         jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
1467         jobdescr->seq_in.command.word =
1468                 cpu_to_caam32(jobdescr->seq_in.command.word);
1469         jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
1470         jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
1471         jobdescr->load_dpovrd.command.word =
1472                 cpu_to_caam32(jobdescr->load_dpovrd.command.word);
1473         jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
1474 #endif
1475
1476         /* Set ptr in input ring to current descriptor  */
1477         sec_write_addr(&ring->input_ring[ring->pidx],
1478                         (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
1479         rte_smp_wmb();
1480
1481         /* Notify HW that a new job is enqueued */
1482         hw_enqueue_desc_on_job_ring(ring);
1483
1484         /* increment the producer index for the current job ring */
1485         ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
1486
1487         return 0;
1488 }
1489
1490 static uint16_t
1491 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1492                        uint16_t nb_ops)
1493 {
1494         /* Function to transmit the frames to given device and queuepair */
1495         uint32_t loop;
1496         int32_t ret;
1497         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
1498         uint16_t num_tx = 0;
1499
1500         PMD_INIT_FUNC_TRACE();
1501         /*Prepare each packet which is to be sent*/
1502         for (loop = 0; loop < nb_ops; loop++) {
1503                 ret = caam_jr_enqueue_op(ops[loop], jr_qp);
1504                 if (!ret)
1505                         num_tx++;
1506         }
1507
1508         jr_qp->tx_pkts += num_tx;
1509
1510         return num_tx;
1511 }
1512
1513 /* Release queue pair */
1514 static int
1515 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
1516                            uint16_t qp_id)
1517 {
1518         struct sec_job_ring_t *internals;
1519         struct caam_jr_qp *qp = NULL;
1520
1521         PMD_INIT_FUNC_TRACE();
1522         CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
1523
1524         internals = dev->data->dev_private;
1525         if (qp_id >= internals->max_nb_queue_pairs) {
1526                 CAAM_JR_ERR("Max supported qpid %d",
1527                              internals->max_nb_queue_pairs);
1528                 return -EINVAL;
1529         }
1530
1531         qp = &internals->qps[qp_id];
1532         qp->ring = NULL;
1533         dev->data->queue_pairs[qp_id] = NULL;
1534
1535         return 0;
1536 }
1537
1538 /* Setup a queue pair */
1539 static int
1540 caam_jr_queue_pair_setup(
1541                 struct rte_cryptodev *dev, uint16_t qp_id,
1542                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1543                 __rte_unused int socket_id,
1544                 __rte_unused struct rte_mempool *session_pool)
1545 {
1546         struct sec_job_ring_t *internals;
1547         struct caam_jr_qp *qp = NULL;
1548
1549         PMD_INIT_FUNC_TRACE();
1550         CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1551
1552         internals = dev->data->dev_private;
1553         if (qp_id >= internals->max_nb_queue_pairs) {
1554                 CAAM_JR_ERR("Max supported qpid %d",
1555                              internals->max_nb_queue_pairs);
1556                 return -EINVAL;
1557         }
1558
1559         qp = &internals->qps[qp_id];
1560         qp->ring = internals;
1561         dev->data->queue_pairs[qp_id] = qp;
1562
1563         return 0;
1564 }
1565
1566 /* Return the number of allocated queue pairs */
1567 static uint32_t
1568 caam_jr_queue_pair_count(struct rte_cryptodev *dev)
1569 {
1570         PMD_INIT_FUNC_TRACE();
1571
1572         return dev->data->nb_queue_pairs;
1573 }
1574
1575 /* Returns the size of the aesni gcm session structure */
1576 static unsigned int
1577 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1578 {
1579         PMD_INIT_FUNC_TRACE();
1580
1581         return sizeof(struct caam_jr_session);
1582 }
1583
1584 static int
1585 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
1586                     struct rte_crypto_sym_xform *xform,
1587                     struct caam_jr_session *session)
1588 {
1589         PMD_INIT_FUNC_TRACE();
1590         session->cipher_alg = xform->cipher.algo;
1591         session->iv.length = xform->cipher.iv.length;
1592         session->iv.offset = xform->cipher.iv.offset;
1593         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1594                                                RTE_CACHE_LINE_SIZE);
1595         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1596                 CAAM_JR_ERR("No Memory for cipher key\n");
1597                 return -ENOMEM;
1598         }
1599         session->cipher_key.length = xform->cipher.key.length;
1600
1601         memcpy(session->cipher_key.data, xform->cipher.key.data,
1602                xform->cipher.key.length);
1603         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1604                         DIR_ENC : DIR_DEC;
1605
1606         return 0;
1607 }
1608
1609 static int
1610 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
1611                   struct rte_crypto_sym_xform *xform,
1612                   struct caam_jr_session *session)
1613 {
1614         PMD_INIT_FUNC_TRACE();
1615         session->auth_alg = xform->auth.algo;
1616         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1617                                              RTE_CACHE_LINE_SIZE);
1618         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1619                 CAAM_JR_ERR("No Memory for auth key\n");
1620                 return -ENOMEM;
1621         }
1622         session->auth_key.length = xform->auth.key.length;
1623         session->digest_length = xform->auth.digest_length;
1624
1625         memcpy(session->auth_key.data, xform->auth.key.data,
1626                xform->auth.key.length);
1627         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1628                         DIR_ENC : DIR_DEC;
1629
1630         return 0;
1631 }
1632
1633 static int
1634 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
1635                   struct rte_crypto_sym_xform *xform,
1636                   struct caam_jr_session *session)
1637 {
1638         PMD_INIT_FUNC_TRACE();
1639         session->aead_alg = xform->aead.algo;
1640         session->iv.length = xform->aead.iv.length;
1641         session->iv.offset = xform->aead.iv.offset;
1642         session->auth_only_len = xform->aead.aad_length;
1643         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1644                                              RTE_CACHE_LINE_SIZE);
1645         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1646                 CAAM_JR_ERR("No Memory for aead key\n");
1647                 return -ENOMEM;
1648         }
1649         session->aead_key.length = xform->aead.key.length;
1650         session->digest_length = xform->aead.digest_length;
1651
1652         memcpy(session->aead_key.data, xform->aead.key.data,
1653                xform->aead.key.length);
1654         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1655                         DIR_ENC : DIR_DEC;
1656
1657         return 0;
1658 }
1659
1660 static int
1661 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
1662                                struct rte_crypto_sym_xform *xform, void *sess)
1663 {
1664         struct sec_job_ring_t *internals = dev->data->dev_private;
1665         struct caam_jr_session *session = sess;
1666
1667         PMD_INIT_FUNC_TRACE();
1668
1669         if (unlikely(sess == NULL)) {
1670                 CAAM_JR_ERR("invalid session struct");
1671                 return -EINVAL;
1672         }
1673
1674         /* Default IV length = 0 */
1675         session->iv.length = 0;
1676
1677         /* Cipher Only */
1678         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1679                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1680                 caam_jr_cipher_init(dev, xform, session);
1681
1682         /* Authentication Only */
1683         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1684                    xform->next == NULL) {
1685                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1686                 caam_jr_auth_init(dev, xform, session);
1687
1688         /* Cipher then Authenticate */
1689         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1690                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1691                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1692                         caam_jr_cipher_init(dev, xform, session);
1693                         caam_jr_auth_init(dev, xform->next, session);
1694                 } else {
1695                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1696                         goto err1;
1697                 }
1698
1699         /* Authenticate then Cipher */
1700         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1701                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1702                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1703                         caam_jr_auth_init(dev, xform, session);
1704                         caam_jr_cipher_init(dev, xform->next, session);
1705                 } else {
1706                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1707                         goto err1;
1708                 }
1709
1710         /* AEAD operation for AES-GCM kind of Algorithms */
1711         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1712                    xform->next == NULL) {
1713                 caam_jr_aead_init(dev, xform, session);
1714
1715         } else {
1716                 CAAM_JR_ERR("Invalid crypto type");
1717                 return -EINVAL;
1718         }
1719         session->ctx_pool = internals->ctx_pool;
1720
1721         return 0;
1722
1723 err1:
1724         rte_free(session->cipher_key.data);
1725         rte_free(session->auth_key.data);
1726         memset(session, 0, sizeof(struct caam_jr_session));
1727
1728         return -EINVAL;
1729 }
1730
1731 static int
1732 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
1733                               struct rte_crypto_sym_xform *xform,
1734                               struct rte_cryptodev_sym_session *sess,
1735                               struct rte_mempool *mempool)
1736 {
1737         void *sess_private_data;
1738         int ret;
1739
1740         PMD_INIT_FUNC_TRACE();
1741
1742         if (rte_mempool_get(mempool, &sess_private_data)) {
1743                 CAAM_JR_ERR("Couldn't get object from session mempool");
1744                 return -ENOMEM;
1745         }
1746
1747         memset(sess_private_data, 0, sizeof(struct caam_jr_session));
1748         ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
1749         if (ret != 0) {
1750                 CAAM_JR_ERR("failed to configure session parameters");
1751                 /* Return session to mempool */
1752                 rte_mempool_put(mempool, sess_private_data);
1753                 return ret;
1754         }
1755
1756         set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
1757
1758         return 0;
1759 }
1760
1761 /* Clear the memory of session so it doesn't leave key material behind */
1762 static void
1763 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
1764                 struct rte_cryptodev_sym_session *sess)
1765 {
1766         uint8_t index = dev->driver_id;
1767         void *sess_priv = get_sym_session_private_data(sess, index);
1768         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1769
1770         PMD_INIT_FUNC_TRACE();
1771
1772         if (sess_priv) {
1773                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1774
1775                 rte_free(s->cipher_key.data);
1776                 rte_free(s->auth_key.data);
1777                 memset(s, 0, sizeof(struct caam_jr_session));
1778                 set_sym_session_private_data(sess, index, NULL);
1779                 rte_mempool_put(sess_mp, sess_priv);
1780         }
1781 }
1782
1783 static int
1784 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1785                           struct rte_security_session_conf *conf,
1786                           void *sess)
1787 {
1788         struct sec_job_ring_t *internals = dev->data->dev_private;
1789         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1790         struct rte_crypto_auth_xform *auth_xform;
1791         struct rte_crypto_cipher_xform *cipher_xform;
1792         struct caam_jr_session *session = (struct caam_jr_session *)sess;
1793
1794         PMD_INIT_FUNC_TRACE();
1795
1796         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1797                 cipher_xform = &conf->crypto_xform->cipher;
1798                 auth_xform = &conf->crypto_xform->next->auth;
1799         } else {
1800                 auth_xform = &conf->crypto_xform->auth;
1801                 cipher_xform = &conf->crypto_xform->next->cipher;
1802         }
1803         session->proto_alg = conf->protocol;
1804         session->cipher_key.data = rte_zmalloc(NULL,
1805                                                cipher_xform->key.length,
1806                                                RTE_CACHE_LINE_SIZE);
1807         if (session->cipher_key.data == NULL &&
1808                         cipher_xform->key.length > 0) {
1809                 CAAM_JR_ERR("No Memory for cipher key\n");
1810                 return -ENOMEM;
1811         }
1812
1813         session->cipher_key.length = cipher_xform->key.length;
1814         session->auth_key.data = rte_zmalloc(NULL,
1815                                         auth_xform->key.length,
1816                                         RTE_CACHE_LINE_SIZE);
1817         if (session->auth_key.data == NULL &&
1818                         auth_xform->key.length > 0) {
1819                 CAAM_JR_ERR("No Memory for auth key\n");
1820                 rte_free(session->cipher_key.data);
1821                 return -ENOMEM;
1822         }
1823         session->auth_key.length = auth_xform->key.length;
1824         memcpy(session->cipher_key.data, cipher_xform->key.data,
1825                         cipher_xform->key.length);
1826         memcpy(session->auth_key.data, auth_xform->key.data,
1827                         auth_xform->key.length);
1828
1829         switch (auth_xform->algo) {
1830         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1831                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1832                 break;
1833         case RTE_CRYPTO_AUTH_MD5_HMAC:
1834                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1835                 break;
1836         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1837                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1838                 break;
1839         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1840                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1841                 break;
1842         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1843                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1844                 break;
1845         case RTE_CRYPTO_AUTH_AES_CMAC:
1846                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1847                 break;
1848         case RTE_CRYPTO_AUTH_NULL:
1849                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1850                 break;
1851         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1852         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1853         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1854         case RTE_CRYPTO_AUTH_SHA1:
1855         case RTE_CRYPTO_AUTH_SHA256:
1856         case RTE_CRYPTO_AUTH_SHA512:
1857         case RTE_CRYPTO_AUTH_SHA224:
1858         case RTE_CRYPTO_AUTH_SHA384:
1859         case RTE_CRYPTO_AUTH_MD5:
1860         case RTE_CRYPTO_AUTH_AES_GMAC:
1861         case RTE_CRYPTO_AUTH_KASUMI_F9:
1862         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1863         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1864                 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
1865                         auth_xform->algo);
1866                 goto out;
1867         default:
1868                 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
1869                         auth_xform->algo);
1870                 goto out;
1871         }
1872
1873         switch (cipher_xform->algo) {
1874         case RTE_CRYPTO_CIPHER_AES_CBC:
1875                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1876                 break;
1877         case RTE_CRYPTO_CIPHER_3DES_CBC:
1878                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1879                 break;
1880         case RTE_CRYPTO_CIPHER_AES_CTR:
1881                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1882                 break;
1883         case RTE_CRYPTO_CIPHER_NULL:
1884         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1885         case RTE_CRYPTO_CIPHER_3DES_ECB:
1886         case RTE_CRYPTO_CIPHER_AES_ECB:
1887         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1888                 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
1889                         cipher_xform->algo);
1890                 goto out;
1891         default:
1892                 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
1893                         cipher_xform->algo);
1894                 goto out;
1895         }
1896
1897         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1898                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
1899                                 sizeof(session->ip4_hdr));
1900                 session->ip4_hdr.ip_v = IPVERSION;
1901                 session->ip4_hdr.ip_hl = 5;
1902                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
1903                                                 sizeof(session->ip4_hdr));
1904                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
1905                 session->ip4_hdr.ip_id = 0;
1906                 session->ip4_hdr.ip_off = 0;
1907                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
1908                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
1909                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
1910                                 : IPPROTO_AH;
1911                 session->ip4_hdr.ip_sum = 0;
1912                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
1913                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
1914                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
1915                                                 (void *)&session->ip4_hdr,
1916                                                 sizeof(struct ip));
1917
1918                 session->encap_pdb.options =
1919                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
1920                         PDBOPTS_ESP_OIHI_PDB_INL |
1921                         PDBOPTS_ESP_IVSRC |
1922                         PDBHMO_ESP_ENCAP_DTTL;
1923                 session->encap_pdb.spi = ipsec_xform->spi;
1924                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
1925
1926                 session->dir = DIR_ENC;
1927         } else if (ipsec_xform->direction ==
1928                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
1929                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
1930                 session->decap_pdb.options = sizeof(struct ip) << 16;
1931                 session->dir = DIR_DEC;
1932         } else
1933                 goto out;
1934         session->ctx_pool = internals->ctx_pool;
1935
1936         return 0;
1937 out:
1938         rte_free(session->auth_key.data);
1939         rte_free(session->cipher_key.data);
1940         memset(session, 0, sizeof(struct caam_jr_session));
1941         return -1;
1942 }
1943
1944 static int
1945 caam_jr_security_session_create(void *dev,
1946                                 struct rte_security_session_conf *conf,
1947                                 struct rte_security_session *sess,
1948                                 struct rte_mempool *mempool)
1949 {
1950         void *sess_private_data;
1951         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1952         int ret;
1953
1954         PMD_INIT_FUNC_TRACE();
1955         if (rte_mempool_get(mempool, &sess_private_data)) {
1956                 CAAM_JR_ERR("Couldn't get object from session mempool");
1957                 return -ENOMEM;
1958         }
1959
1960         switch (conf->protocol) {
1961         case RTE_SECURITY_PROTOCOL_IPSEC:
1962                 ret = caam_jr_set_ipsec_session(cdev, conf,
1963                                 sess_private_data);
1964                 break;
1965         case RTE_SECURITY_PROTOCOL_MACSEC:
1966                 return -ENOTSUP;
1967         default:
1968                 return -EINVAL;
1969         }
1970         if (ret != 0) {
1971                 CAAM_JR_ERR("failed to configure session parameters");
1972                 /* Return session to mempool */
1973                 rte_mempool_put(mempool, sess_private_data);
1974                 return ret;
1975         }
1976
1977         set_sec_session_private_data(sess, sess_private_data);
1978
1979         return ret;
1980 }
1981
1982 /* Clear the memory of session so it doesn't leave key material behind */
1983 static int
1984 caam_jr_security_session_destroy(void *dev __rte_unused,
1985                                  struct rte_security_session *sess)
1986 {
1987         PMD_INIT_FUNC_TRACE();
1988         void *sess_priv = get_sec_session_private_data(sess);
1989
1990         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1991
1992         if (sess_priv) {
1993                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1994
1995                 rte_free(s->cipher_key.data);
1996                 rte_free(s->auth_key.data);
1997                 memset(sess, 0, sizeof(struct caam_jr_session));
1998                 set_sec_session_private_data(sess, NULL);
1999                 rte_mempool_put(sess_mp, sess_priv);
2000         }
2001         return 0;
2002 }
2003
2004
2005 static int
2006 caam_jr_dev_configure(struct rte_cryptodev *dev,
2007                        struct rte_cryptodev_config *config __rte_unused)
2008 {
2009         char str[20];
2010         struct sec_job_ring_t *internals;
2011
2012         PMD_INIT_FUNC_TRACE();
2013
2014         internals = dev->data->dev_private;
2015         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2016         if (!internals->ctx_pool) {
2017                 internals->ctx_pool = rte_mempool_create((const char *)str,
2018                                                 CTX_POOL_NUM_BUFS,
2019                                                 sizeof(struct caam_jr_op_ctx),
2020                                                 CTX_POOL_CACHE_SIZE, 0,
2021                                                 NULL, NULL, NULL, NULL,
2022                                                 SOCKET_ID_ANY, 0);
2023                 if (!internals->ctx_pool) {
2024                         CAAM_JR_ERR("%s create failed\n", str);
2025                         return -ENOMEM;
2026                 }
2027         } else
2028                 CAAM_JR_INFO("mempool already created for dev_id : %d",
2029                                 dev->data->dev_id);
2030
2031         return 0;
2032 }
2033
2034 static int
2035 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
2036 {
2037         PMD_INIT_FUNC_TRACE();
2038         return 0;
2039 }
2040
2041 static void
2042 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
2043 {
2044         PMD_INIT_FUNC_TRACE();
2045 }
2046
2047 static int
2048 caam_jr_dev_close(struct rte_cryptodev *dev)
2049 {
2050         struct sec_job_ring_t *internals;
2051
2052         PMD_INIT_FUNC_TRACE();
2053
2054         if (dev == NULL)
2055                 return -ENOMEM;
2056
2057         internals = dev->data->dev_private;
2058         rte_mempool_free(internals->ctx_pool);
2059         internals->ctx_pool = NULL;
2060
2061         return 0;
2062 }
2063
2064 static void
2065 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
2066                        struct rte_cryptodev_info *info)
2067 {
2068         struct sec_job_ring_t *internals = dev->data->dev_private;
2069
2070         PMD_INIT_FUNC_TRACE();
2071         if (info != NULL) {
2072                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2073                 info->feature_flags = dev->feature_flags;
2074                 info->capabilities = caam_jr_get_cryptodev_capabilities();
2075                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2076                 info->driver_id = cryptodev_driver_id;
2077         }
2078 }
2079
2080 static struct rte_cryptodev_ops caam_jr_ops = {
2081         .dev_configure        = caam_jr_dev_configure,
2082         .dev_start            = caam_jr_dev_start,
2083         .dev_stop             = caam_jr_dev_stop,
2084         .dev_close            = caam_jr_dev_close,
2085         .dev_infos_get        = caam_jr_dev_infos_get,
2086         .stats_get            = caam_jr_stats_get,
2087         .stats_reset          = caam_jr_stats_reset,
2088         .queue_pair_setup     = caam_jr_queue_pair_setup,
2089         .queue_pair_release   = caam_jr_queue_pair_release,
2090         .queue_pair_count     = caam_jr_queue_pair_count,
2091         .sym_session_get_size = caam_jr_sym_session_get_size,
2092         .sym_session_configure = caam_jr_sym_session_configure,
2093         .sym_session_clear    = caam_jr_sym_session_clear
2094 };
2095
2096 static struct rte_security_ops caam_jr_security_ops = {
2097         .session_create = caam_jr_security_session_create,
2098         .session_update = NULL,
2099         .session_stats_get = NULL,
2100         .session_destroy = caam_jr_security_session_destroy,
2101         .set_pkt_metadata = NULL,
2102         .capabilities_get = caam_jr_get_security_capabilities
2103 };
2104
2105 /* @brief Flush job rings of any processed descs.
2106  * The processed descs are silently dropped,
2107  * WITHOUT being notified to UA.
2108  */
2109 static void
2110 close_job_ring(struct sec_job_ring_t *job_ring)
2111 {
2112         PMD_INIT_FUNC_TRACE();
2113         if (job_ring->irq_fd) {
2114                 /* Producer index is frozen. If consumer index is not equal
2115                  * with producer index, then we have descs to flush.
2116                  */
2117                 while (job_ring->pidx != job_ring->cidx)
2118                         hw_flush_job_ring(job_ring, false, NULL);
2119
2120                 /* free the uio job ring */
2121                 free_job_ring(job_ring->irq_fd);
2122                 job_ring->irq_fd = 0;
2123                 caam_jr_dma_free(job_ring->input_ring);
2124                 caam_jr_dma_free(job_ring->output_ring);
2125                 g_job_rings_no--;
2126         }
2127 }
2128
2129 /** @brief Release the software and hardware resources tied to a job ring.
2130  * @param [in] job_ring The job ring
2131  *
2132  * @retval  0 for success
2133  * @retval  -1 for error
2134  */
2135 static int
2136 shutdown_job_ring(struct sec_job_ring_t *job_ring)
2137 {
2138         int ret = 0;
2139
2140         PMD_INIT_FUNC_TRACE();
2141         ASSERT(job_ring != NULL);
2142         ret = hw_shutdown_job_ring(job_ring);
2143         SEC_ASSERT(ret == 0, ret,
2144                 "Failed to shutdown hardware job ring %p",
2145                 job_ring);
2146
2147         if (job_ring->coalescing_en)
2148                 hw_job_ring_disable_coalescing(job_ring);
2149
2150         if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
2151                 ret = caam_jr_disable_irqs(job_ring->irq_fd);
2152                 SEC_ASSERT(ret == 0, ret,
2153                 "Failed to disable irqs for job ring %p",
2154                 job_ring);
2155         }
2156
2157         return ret;
2158 }
2159
2160 /*
2161  * @brief Release the resources used by the SEC user space driver.
2162  *
2163  * Reset and release SEC's job rings indicated by the User Application at
2164  * init_job_ring() and free any memory allocated internally.
2165  * Call once during application tear down.
2166  *
2167  * @note In case there are any descriptors in-flight (descriptors received by
2168  * SEC driver for processing and for which no response was yet provided to UA),
2169  * the descriptors are discarded without any notifications to User Application.
2170  *
2171  * @retval ::0                  is returned for a successful execution
2172  * @retval ::-1         is returned if SEC driver release is in progress
2173  */
2174 static int
2175 caam_jr_dev_uninit(struct rte_cryptodev *dev)
2176 {
2177         struct sec_job_ring_t *internals;
2178
2179         PMD_INIT_FUNC_TRACE();
2180         if (dev == NULL)
2181                 return -ENODEV;
2182
2183         internals = dev->data->dev_private;
2184         rte_free(dev->security_ctx);
2185
2186         /* If any descriptors in flight , poll and wait
2187          * until all descriptors are received and silently discarded.
2188          */
2189         if (internals) {
2190                 shutdown_job_ring(internals);
2191                 close_job_ring(internals);
2192                 rte_mempool_free(internals->ctx_pool);
2193         }
2194
2195         CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
2196
2197         /* last caam jr instance) */
2198         if (g_job_rings_no == 0)
2199                 g_driver_state = SEC_DRIVER_STATE_IDLE;
2200
2201         return SEC_SUCCESS;
2202 }
2203
2204 /* @brief Initialize the software and hardware resources tied to a job ring.
2205  * @param [in] jr_mode;         Model to be used by SEC Driver to receive
2206  *                              notifications from SEC.  Can be either
2207  *                              of the three: #SEC_NOTIFICATION_TYPE_NAPI
2208  *                              #SEC_NOTIFICATION_TYPE_IRQ or
2209  *                              #SEC_NOTIFICATION_TYPE_POLL
2210  * @param [in] NAPI_mode        The NAPI work mode to configure a job ring at
2211  *                              startup. Used only when #SEC_NOTIFICATION_TYPE
2212  *                              is set to #SEC_NOTIFICATION_TYPE_NAPI.
2213  * @param [in] irq_coalescing_timer This value determines the maximum
2214  *                                      amount of time after processing a
2215  *                                      descriptor before raising an interrupt.
2216  * @param [in] irq_coalescing_count This value determines how many
2217  *                                      descriptors are completed before
2218  *                                      raising an interrupt.
2219  * @param [in] reg_base_addr,   The job ring base address register
2220  * @param [in] irq_id           The job ring interrupt identification number.
2221  * @retval  job_ring_handle for successful job ring configuration
2222  * @retval  NULL on error
2223  *
2224  */
2225 static void *
2226 init_job_ring(void *reg_base_addr, uint32_t irq_id)
2227 {
2228         struct sec_job_ring_t *job_ring = NULL;
2229         int i, ret = 0;
2230         int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
2231         int napi_mode = 0;
2232         int irq_coalescing_timer = 0;
2233         int irq_coalescing_count = 0;
2234
2235         for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
2236                 if (g_job_rings[i].irq_fd == 0) {
2237                         job_ring = &g_job_rings[i];
2238                         g_job_rings_no++;
2239                         break;
2240                 }
2241         }
2242         if (job_ring == NULL) {
2243                 CAAM_JR_ERR("No free job ring\n");
2244                 return NULL;
2245         }
2246
2247         job_ring->register_base_addr = reg_base_addr;
2248         job_ring->jr_mode = jr_mode;
2249         job_ring->napi_mode = 0;
2250         job_ring->irq_fd = irq_id;
2251
2252         /* Allocate mem for input and output ring */
2253
2254         /* Allocate memory for input ring */
2255         job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2256                                 SEC_DMA_MEM_INPUT_RING_SIZE);
2257         memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
2258
2259         /* Allocate memory for output ring */
2260         job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2261                                 SEC_DMA_MEM_OUTPUT_RING_SIZE);
2262         memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
2263
2264         /* Reset job ring in SEC hw and configure job ring registers */
2265         ret = hw_reset_job_ring(job_ring);
2266         if (ret != 0) {
2267                 CAAM_JR_ERR("Failed to reset hardware job ring");
2268                 goto cleanup;
2269         }
2270
2271         if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
2272         /* When SEC US driver works in NAPI mode, the UA can select
2273          * if the driver starts with IRQs on or off.
2274          */
2275                 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
2276                         CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
2277                                 job_ring);
2278                         ret = caam_jr_enable_irqs(job_ring->irq_fd);
2279                         if (ret != 0) {
2280                                 CAAM_JR_ERR("Failed to enable irqs for job ring");
2281                                 goto cleanup;
2282                         }
2283                 }
2284         } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
2285         /* When SEC US driver works in pure interrupt mode,
2286          * IRQ's are always enabled.
2287          */
2288                 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
2289                          job_ring);
2290                 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2291                 if (ret != 0) {
2292                         CAAM_JR_ERR("Failed to enable irqs for job ring");
2293                         goto cleanup;
2294                 }
2295         }
2296         if (irq_coalescing_timer || irq_coalescing_count) {
2297                 hw_job_ring_set_coalescing_param(job_ring,
2298                          irq_coalescing_timer,
2299                          irq_coalescing_count);
2300
2301                 hw_job_ring_enable_coalescing(job_ring);
2302                 job_ring->coalescing_en = 1;
2303         }
2304
2305         job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
2306         job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
2307         job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
2308
2309         return job_ring;
2310 cleanup:
2311         caam_jr_dma_free(job_ring->output_ring);
2312         caam_jr_dma_free(job_ring->input_ring);
2313         return NULL;
2314 }
2315
2316
2317 static int
2318 caam_jr_dev_init(const char *name,
2319                  struct rte_vdev_device *vdev,
2320                  struct rte_cryptodev_pmd_init_params *init_params)
2321 {
2322         struct rte_cryptodev *dev;
2323         struct rte_security_ctx *security_instance;
2324         struct uio_job_ring *job_ring;
2325         char str[RTE_CRYPTODEV_NAME_MAX_LEN];
2326
2327         PMD_INIT_FUNC_TRACE();
2328
2329         /* Validate driver state */
2330         if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
2331                 g_job_rings_max = sec_configure();
2332                 if (!g_job_rings_max) {
2333                         CAAM_JR_ERR("No job ring detected on UIO !!!!");
2334                         return -1;
2335                 }
2336                 /* Update driver state */
2337                 g_driver_state = SEC_DRIVER_STATE_STARTED;
2338         }
2339
2340         if (g_job_rings_no >= g_job_rings_max) {
2341                 CAAM_JR_ERR("No more job rings available max=%d!!!!",
2342                                 g_job_rings_max);
2343                 return -1;
2344         }
2345
2346         job_ring = config_job_ring();
2347         if (job_ring == NULL) {
2348                 CAAM_JR_ERR("failed to create job ring");
2349                 goto init_error;
2350         }
2351
2352         snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
2353
2354         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2355         if (dev == NULL) {
2356                 CAAM_JR_ERR("failed to create cryptodev vdev");
2357                 goto cleanup;
2358         }
2359         /*TODO free it during teardown*/
2360         dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
2361                                                 job_ring->uio_fd);
2362
2363         if (!dev->data->dev_private) {
2364                 CAAM_JR_ERR("Ring memory allocation failed\n");
2365                 goto cleanup2;
2366         }
2367
2368         dev->driver_id = cryptodev_driver_id;
2369         dev->dev_ops = &caam_jr_ops;
2370
2371         /* register rx/tx burst functions for data path */
2372         dev->dequeue_burst = caam_jr_dequeue_burst;
2373         dev->enqueue_burst = caam_jr_enqueue_burst;
2374         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2375                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2376                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2377                         RTE_CRYPTODEV_FF_SECURITY |
2378                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2379                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2380                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2381                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2382                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2383
2384         /* For secondary processes, we don't initialise any further as primary
2385          * has already done this work. Only check we don't need a different
2386          * RX function
2387          */
2388         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2389                 CAAM_JR_WARN("Device already init by primary process");
2390                 return 0;
2391         }
2392
2393         /*TODO free it during teardown*/
2394         security_instance = rte_malloc("caam_jr",
2395                                 sizeof(struct rte_security_ctx), 0);
2396         if (security_instance == NULL) {
2397                 CAAM_JR_ERR("memory allocation failed\n");
2398                 //todo error handling.
2399                 goto cleanup2;
2400         }
2401
2402         security_instance->device = (void *)dev;
2403         security_instance->ops = &caam_jr_security_ops;
2404         security_instance->sess_cnt = 0;
2405         dev->security_ctx = security_instance;
2406
2407         RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
2408
2409         return 0;
2410
2411 cleanup2:
2412         caam_jr_dev_uninit(dev);
2413         rte_cryptodev_pmd_release_device(dev);
2414 cleanup:
2415         free_job_ring(job_ring->uio_fd);
2416 init_error:
2417         CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
2418                         init_params->name);
2419
2420         return -ENXIO;
2421 }
2422
2423 /** Initialise CAAM JR crypto device */
2424 static int
2425 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
2426 {
2427         struct rte_cryptodev_pmd_init_params init_params = {
2428                 "",
2429                 sizeof(struct sec_job_ring_t),
2430                 rte_socket_id(),
2431                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2432         };
2433         const char *name;
2434         const char *input_args;
2435
2436         name = rte_vdev_device_name(vdev);
2437         if (name == NULL)
2438                 return -EINVAL;
2439
2440         input_args = rte_vdev_device_args(vdev);
2441         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2442
2443         /* if sec device version is not configured */
2444         if (!rta_get_sec_era()) {
2445                 const struct device_node *caam_node;
2446
2447                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2448                         const uint32_t *prop = of_get_property(caam_node,
2449                                         "fsl,sec-era",
2450                                         NULL);
2451                         if (prop) {
2452                                 rta_set_sec_era(
2453                                         INTL_SEC_ERA(cpu_to_caam32(*prop)));
2454                                 break;
2455                         }
2456                 }
2457         }
2458 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
2459         if (rta_get_sec_era() > RTA_SEC_ERA_8) {
2460                 RTE_LOG(ERR, PMD,
2461                 "CAAM is compiled in BE mode for device with sec era > 8???\n");
2462                 return -EINVAL;
2463         }
2464 #endif
2465
2466         return caam_jr_dev_init(name, vdev, &init_params);
2467 }
2468
2469 /** Uninitialise CAAM JR crypto device */
2470 static int
2471 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
2472 {
2473         struct rte_cryptodev *cryptodev;
2474         const char *name;
2475
2476         name = rte_vdev_device_name(vdev);
2477         if (name == NULL)
2478                 return -EINVAL;
2479
2480         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2481         if (cryptodev == NULL)
2482                 return -ENODEV;
2483
2484         caam_jr_dev_uninit(cryptodev);
2485
2486         return rte_cryptodev_pmd_destroy(cryptodev);
2487 }
2488
2489 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
2490         .probe = cryptodev_caam_jr_probe,
2491         .remove = cryptodev_caam_jr_remove
2492 };
2493
2494 static struct cryptodev_driver caam_jr_crypto_drv;
2495
2496 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
2497 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
2498         "max_nb_queue_pairs=<int>"
2499         "socket_id=<int>");
2500 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
2501                 cryptodev_driver_id);
2502
2503 RTE_INIT(caam_jr_init_log)
2504 {
2505         caam_jr_logtype = rte_log_register("pmd.crypto.caam");
2506         if (caam_jr_logtype >= 0)
2507                 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
2508 }