drivers/crypto/virtio/virtio_cryptodev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
   3  */
   4 #include <stdbool.h>
   5 #include <unistd.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_errno.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_cryptodev.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_eal.h>
  14
  15 #include "virtio_cryptodev.h"
  16 #include "virtqueue.h"
  17 #include "virtio_crypto_algs.h"
  18 #include "virtio_crypto_capabilities.h"
  19
  20 int virtio_crypto_logtype_init;
  21 int virtio_crypto_logtype_session;
  22 int virtio_crypto_logtype_rx;
  23 int virtio_crypto_logtype_tx;
  24 int virtio_crypto_logtype_driver;
  25
  26 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
  27                 struct rte_cryptodev_config *config);
  28 static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
  29 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
  30 static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
  31 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
  32                 struct rte_cryptodev_info *dev_info);
  33 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
  34                 struct rte_cryptodev_stats *stats);
  35 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
  36 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
  37                 uint16_t queue_pair_id,
  38                 const struct rte_cryptodev_qp_conf *qp_conf,
  39                 int socket_id,
  40                 struct rte_mempool *session_pool);
  41 static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
  42                 uint16_t queue_pair_id);
  43 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
  44 static unsigned int virtio_crypto_sym_get_session_private_size(
  45                 struct rte_cryptodev *dev);
  46 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
  47                 struct rte_cryptodev_sym_session *sess);
  48 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
  49                 struct rte_crypto_sym_xform *xform,
  50                 struct rte_cryptodev_sym_session *session,
  51                 struct rte_mempool *mp);
  52
  53 /*
  54  * The set of PCI devices this driver supports
  55  */
  56 static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
  57         { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
  58                                 VIRTIO_CRYPTO_PCI_DEVICEID) },
  59         { .vendor_id = 0, /* sentinel */ },
  60 };
  61
  62 static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
  63         VIRTIO_SYM_CAPABILITIES,
  64         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
  65 };
  66
  67 uint8_t cryptodev_virtio_driver_id;
  68
  69 #define NUM_ENTRY_SYM_CREATE_SESSION 4
  70
  71 static int
  72 virtio_crypto_send_command(struct virtqueue *vq,
  73                 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
  74                 uint8_t *auth_key, struct virtio_crypto_session *session)
  75 {
  76         uint8_t idx = 0;
  77         uint8_t needed = 1;
  78         uint32_t head = 0;
  79         uint32_t len_cipher_key = 0;
  80         uint32_t len_auth_key = 0;
  81         uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
  82         uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
  83         uint32_t len_total = 0;
  84         uint32_t input_offset = 0;
  85         void *virt_addr_started = NULL;
  86         phys_addr_t phys_addr_started;
  87         struct vring_desc *desc;
  88         uint32_t desc_offset;
  89         struct virtio_crypto_session_input *input;
  90         int ret;
  91
  92         PMD_INIT_FUNC_TRACE();
  93
  94         if (session == NULL) {
  95                 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
  96                 return -EINVAL;
  97         }
  98         /* cipher only is supported, it is available if auth_key is NULL */
  99         if (!cipher_key) {
 100                 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
 101                 return -EINVAL;
 102         }
 103
 104         head = vq->vq_desc_head_idx;
 105         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
 106                                         head, vq);
 107
 108         if (vq->vq_free_cnt < needed) {
 109                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
 110                 return -ENOSPC;
 111         }
 112
 113         /* calculate the length of cipher key */
 114         if (cipher_key) {
 115                 switch (ctrl->u.sym_create_session.op_type) {
 116                 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
 117                         len_cipher_key
 118                                 = ctrl->u.sym_create_session.u.cipher
 119                                                         .para.keylen;
 120                         break;
 121                 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
 122                         len_cipher_key
 123                                 = ctrl->u.sym_create_session.u.chain
 124                                         .para.cipher_param.keylen;
 125                         break;
 126                 default:
 127                         VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
 128                         return -EINVAL;
 129                 }
 130         }
 131
 132         /* calculate the length of auth key */
 133         if (auth_key) {
 134                 len_auth_key =
 135                         ctrl->u.sym_create_session.u.chain.para.u.mac_param
 136                                 .auth_key_len;
 137         }
 138
 139         /*
 140          * malloc memory to store indirect vring_desc entries, including
 141          * ctrl request, cipher key, auth key, session input and desc vring
 142          */
 143         desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
 144                 + len_session_input;
 145         virt_addr_started = rte_malloc(NULL,
 146                 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
 147                         * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
 148         if (virt_addr_started == NULL) {
 149                 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
 150                 return -ENOSPC;
 151         }
 152         phys_addr_started = rte_malloc_virt2iova(virt_addr_started);
 153
 154         /* address to store indirect vring desc entries */
 155         desc = (struct vring_desc *)
 156                 ((uint8_t *)virt_addr_started + desc_offset);
 157
 158         /*  ctrl req part */
 159         memcpy(virt_addr_started, ctrl, len_ctrl_req);
 160         desc[idx].addr = phys_addr_started;
 161         desc[idx].len = len_ctrl_req;
 162         desc[idx].flags = VRING_DESC_F_NEXT;
 163         desc[idx].next = idx + 1;
 164         idx++;
 165         len_total += len_ctrl_req;
 166         input_offset += len_ctrl_req;
 167
 168         /* cipher key part */
 169         if (len_cipher_key > 0) {
 170                 memcpy((uint8_t *)virt_addr_started + len_total,
 171                         cipher_key, len_cipher_key);
 172
 173                 desc[idx].addr = phys_addr_started + len_total;
 174                 desc[idx].len = len_cipher_key;
 175                 desc[idx].flags = VRING_DESC_F_NEXT;
 176                 desc[idx].next = idx + 1;
 177                 idx++;
 178                 len_total += len_cipher_key;
 179                 input_offset += len_cipher_key;
 180         }
 181
 182         /* auth key part */
 183         if (len_auth_key > 0) {
 184                 memcpy((uint8_t *)virt_addr_started + len_total,
 185                         auth_key, len_auth_key);
 186
 187                 desc[idx].addr = phys_addr_started + len_total;
 188                 desc[idx].len = len_auth_key;
 189                 desc[idx].flags = VRING_DESC_F_NEXT;
 190                 desc[idx].next = idx + 1;
 191                 idx++;
 192                 len_total += len_auth_key;
 193                 input_offset += len_auth_key;
 194         }
 195
 196         /* input part */
 197         input = (struct virtio_crypto_session_input *)
 198                 ((uint8_t *)virt_addr_started + input_offset);
 199         input->status = VIRTIO_CRYPTO_ERR;
 200         input->session_id = ~0ULL;
 201         desc[idx].addr = phys_addr_started + len_total;
 202         desc[idx].len = len_session_input;
 203         desc[idx].flags = VRING_DESC_F_WRITE;
 204         idx++;
 205
 206         /* use a single desc entry */
 207         vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
 208         vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
 209         vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
 210         vq->vq_free_cnt--;
 211
 212         vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
 213
 214         vq_update_avail_ring(vq, head);
 215         vq_update_avail_idx(vq);
 216
 217         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
 218                                         vq->vq_queue_index);
 219
 220         virtqueue_notify(vq);
 221
 222         rte_rmb();
 223         while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
 224                 rte_rmb();
 225                 usleep(100);
 226         }
 227
 228         while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
 229                 uint32_t idx, desc_idx, used_idx;
 230                 struct vring_used_elem *uep;
 231
 232                 used_idx = (uint32_t)(vq->vq_used_cons_idx
 233                                 & (vq->vq_nentries - 1));
 234                 uep = &vq->vq_ring.used->ring[used_idx];
 235                 idx = (uint32_t) uep->id;
 236                 desc_idx = idx;
 237
 238                 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
 239                         desc_idx = vq->vq_ring.desc[desc_idx].next;
 240                         vq->vq_free_cnt++;
 241                 }
 242
 243                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
 244                 vq->vq_desc_head_idx = idx;
 245
 246                 vq->vq_used_cons_idx++;
 247                 vq->vq_free_cnt++;
 248         }
 249
 250         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
 251                         "vq->vq_desc_head_idx=%d",
 252                         vq->vq_free_cnt, vq->vq_desc_head_idx);
 253
 254         /* get the result */
 255         if (input->status != VIRTIO_CRYPTO_OK) {
 256                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
 257                                 "status=%u, session_id=%" PRIu64 "",
 258                                 input->status, input->session_id);
 259                 rte_free(virt_addr_started);
 260                 ret = -1;
 261         } else {
 262                 session->session_id = input->session_id;
 263
 264                 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
 265                                 "session_id=%" PRIu64 "", input->session_id);
 266                 rte_free(virt_addr_started);
 267                 ret = 0;
 268         }
 269
 270         return ret;
 271 }
 272
 273 void
 274 virtio_crypto_queue_release(struct virtqueue *vq)
 275 {
 276         struct virtio_crypto_hw *hw;
 277
 278         PMD_INIT_FUNC_TRACE();
 279
 280         if (vq) {
 281                 hw = vq->hw;
 282                 /* Select and deactivate the queue */
 283                 VTPCI_OPS(hw)->del_queue(hw, vq);
 284
 285                 rte_memzone_free(vq->mz);
 286                 rte_mempool_free(vq->mpool);
 287                 rte_free(vq);
 288         }
 289 }
 290
 291 #define MPOOL_MAX_NAME_SZ 32
 292
 293 int
 294 virtio_crypto_queue_setup(struct rte_cryptodev *dev,
 295                 int queue_type,
 296                 uint16_t vtpci_queue_idx,
 297                 uint16_t nb_desc,
 298                 int socket_id,
 299                 struct virtqueue **pvq)
 300 {
 301         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
 302         char mpool_name[MPOOL_MAX_NAME_SZ];
 303         const struct rte_memzone *mz;
 304         unsigned int vq_size, size;
 305         struct virtio_crypto_hw *hw = dev->data->dev_private;
 306         struct virtqueue *vq = NULL;
 307         uint32_t i = 0;
 308         uint32_t j;
 309
 310         PMD_INIT_FUNC_TRACE();
 311
 312         VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
 313
 314         /*
 315          * Read the virtqueue size from the Queue Size field
 316          * Always power of 2 and if 0 virtqueue does not exist
 317          */
 318         vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
 319         if (vq_size == 0) {
 320                 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
 321                 return -EINVAL;
 322         }
 323         VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
 324
 325         if (!rte_is_power_of_2(vq_size)) {
 326                 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
 327                 return -EINVAL;
 328         }
 329
 330         if (queue_type == VTCRYPTO_DATAQ) {
 331                 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
 332                                 dev->data->dev_id, vtpci_queue_idx);
 333                 snprintf(mpool_name, sizeof(mpool_name),
 334                                 "dev%d_dataqueue%d_mpool",
 335                                 dev->data->dev_id, vtpci_queue_idx);
 336         } else if (queue_type == VTCRYPTO_CTRLQ) {
 337                 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
 338                                 dev->data->dev_id);
 339                 snprintf(mpool_name, sizeof(mpool_name),
 340                                 "dev%d_controlqueue_mpool",
 341                                 dev->data->dev_id);
 342         }
 343         size = RTE_ALIGN_CEIL(sizeof(*vq) +
 344                                 vq_size * sizeof(struct vq_desc_extra),
 345                                 RTE_CACHE_LINE_SIZE);
 346         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
 347                                 socket_id);
 348         if (vq == NULL) {
 349                 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
 350                 return -ENOMEM;
 351         }
 352
 353         if (queue_type == VTCRYPTO_DATAQ) {
 354                 /* pre-allocate a mempool and use it in the data plane to
 355                  * improve performance
 356                  */
 357                 vq->mpool = rte_mempool_lookup(mpool_name);
 358                 if (vq->mpool == NULL)
 359                         vq->mpool = rte_mempool_create(mpool_name,
 360                                         vq_size,
 361                                         sizeof(struct virtio_crypto_op_cookie),
 362                                         RTE_CACHE_LINE_SIZE, 0,
 363                                         NULL, NULL, NULL, NULL, socket_id,
 364                                         0);
 365                 if (!vq->mpool) {
 366                         VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
 367                                         "Cannot create mempool");
 368                         goto mpool_create_err;
 369                 }
 370                 for (i = 0; i < vq_size; i++) {
 371                         vq->vq_descx[i].cookie =
 372                                 rte_zmalloc("crypto PMD op cookie pointer",
 373                                         sizeof(struct virtio_crypto_op_cookie),
 374                                         RTE_CACHE_LINE_SIZE);
 375                         if (vq->vq_descx[i].cookie == NULL) {
 376                                 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
 377                                                 "alloc mem for cookie");
 378                                 goto cookie_alloc_err;
 379                         }
 380                 }
 381         }
 382
 383         vq->hw = hw;
 384         vq->dev_id = dev->data->dev_id;
 385         vq->vq_queue_index = vtpci_queue_idx;
 386         vq->vq_nentries = vq_size;
 387
 388         /*
 389          * Using part of the vring entries is permitted, but the maximum
 390          * is vq_size
 391          */
 392         if (nb_desc == 0 || nb_desc > vq_size)
 393                 nb_desc = vq_size;
 394         vq->vq_free_cnt = nb_desc;
 395
 396         /*
 397          * Reserve a memzone for vring elements
 398          */
 399         size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
 400         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
 401         VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
 402                         (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
 403                         size, vq->vq_ring_size);
 404
 405         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
 406                         socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
 407         if (mz == NULL) {
 408                 if (rte_errno == EEXIST)
 409                         mz = rte_memzone_lookup(vq_name);
 410                 if (mz == NULL) {
 411                         VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
 412                         goto mz_reserve_err;
 413                 }
 414         }
 415
 416         /*
 417          * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
 418          * and only accepts 32 bit page frame number.
 419          * Check if the allocated physical memory exceeds 16TB.
 420          */
 421         if ((mz->phys_addr + vq->vq_ring_size - 1)
 422                                 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
 423                 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
 424                                         "above 16TB!");
 425                 goto vring_addr_err;
 426         }
 427
 428         memset(mz->addr, 0, sizeof(mz->len));
 429         vq->mz = mz;
 430         vq->vq_ring_mem = mz->phys_addr;
 431         vq->vq_ring_virt_mem = mz->addr;
 432         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
 433                                         (uint64_t)mz->phys_addr);
 434         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
 435                                         (uint64_t)(uintptr_t)mz->addr);
 436
 437         *pvq = vq;
 438
 439         return 0;
 440
 441 vring_addr_err:
 442         rte_memzone_free(mz);
 443 mz_reserve_err:
 444 cookie_alloc_err:
 445         rte_mempool_free(vq->mpool);
 446         if (i != 0) {
 447                 for (j = 0; j < i; j++)
 448                         rte_free(vq->vq_descx[j].cookie);
 449         }
 450 mpool_create_err:
 451         rte_free(vq);
 452         return -ENOMEM;
 453 }
 454
 455 static int
 456 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
 457 {
 458         int ret;
 459         struct virtqueue *vq;
 460         struct virtio_crypto_hw *hw = dev->data->dev_private;
 461
 462         /* if virtio device has started, do not touch the virtqueues */
 463         if (dev->data->dev_started)
 464                 return 0;
 465
 466         PMD_INIT_FUNC_TRACE();
 467
 468         ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
 469                         0, SOCKET_ID_ANY, &vq);
 470         if (ret < 0) {
 471                 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
 472                 return ret;
 473         }
 474
 475         hw->cvq = vq;
 476
 477         return 0;
 478 }
 479
 480 static void
 481 virtio_crypto_free_queues(struct rte_cryptodev *dev)
 482 {
 483         unsigned int i;
 484         struct virtio_crypto_hw *hw = dev->data->dev_private;
 485
 486         PMD_INIT_FUNC_TRACE();
 487
 488         /* control queue release */
 489         virtio_crypto_queue_release(hw->cvq);
 490
 491         /* data queue release */
 492         for (i = 0; i < hw->max_dataqueues; i++)
 493                 virtio_crypto_queue_release(dev->data->queue_pairs[i]);
 494 }
 495
 496 static int
 497 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
 498 {
 499         return 0;
 500 }
 501
 502 /*
 503  * dev_ops for virtio, bare necessities for basic operation
 504  */
 505 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
 506         /* Device related operations */
 507         .dev_configure                   = virtio_crypto_dev_configure,
 508         .dev_start                       = virtio_crypto_dev_start,
 509         .dev_stop                        = virtio_crypto_dev_stop,
 510         .dev_close                       = virtio_crypto_dev_close,
 511         .dev_infos_get                   = virtio_crypto_dev_info_get,
 512
 513         .stats_get                       = virtio_crypto_dev_stats_get,
 514         .stats_reset                     = virtio_crypto_dev_stats_reset,
 515
 516         .queue_pair_setup                = virtio_crypto_qp_setup,
 517         .queue_pair_release              = virtio_crypto_qp_release,
 518         .queue_pair_count                = NULL,
 519
 520         /* Crypto related operations */
 521         .sym_session_get_size           = virtio_crypto_sym_get_session_private_size,
 522         .sym_session_configure          = virtio_crypto_sym_configure_session,
 523         .sym_session_clear              = virtio_crypto_sym_clear_session
 524 };
 525
 526 static void
 527 virtio_crypto_update_stats(struct rte_cryptodev *dev,
 528                 struct rte_cryptodev_stats *stats)
 529 {
 530         unsigned int i;
 531         struct virtio_crypto_hw *hw = dev->data->dev_private;
 532
 533         PMD_INIT_FUNC_TRACE();
 534
 535         if (stats == NULL) {
 536                 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
 537                 return;
 538         }
 539
 540         for (i = 0; i < hw->max_dataqueues; i++) {
 541                 const struct virtqueue *data_queue
 542                         = dev->data->queue_pairs[i];
 543                 if (data_queue == NULL)
 544                         continue;
 545
 546                 stats->enqueued_count += data_queue->packets_sent_total;
 547                 stats->enqueue_err_count += data_queue->packets_sent_failed;
 548
 549                 stats->dequeued_count += data_queue->packets_received_total;
 550                 stats->dequeue_err_count
 551                         += data_queue->packets_received_failed;
 552         }
 553 }
 554
 555 static void
 556 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
 557                 struct rte_cryptodev_stats *stats)
 558 {
 559         PMD_INIT_FUNC_TRACE();
 560
 561         virtio_crypto_update_stats(dev, stats);
 562 }
 563
 564 static void
 565 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
 566 {
 567         unsigned int i;
 568         struct virtio_crypto_hw *hw = dev->data->dev_private;
 569
 570         PMD_INIT_FUNC_TRACE();
 571
 572         for (i = 0; i < hw->max_dataqueues; i++) {
 573                 struct virtqueue *data_queue = dev->data->queue_pairs[i];
 574                 if (data_queue == NULL)
 575                         continue;
 576
 577                 data_queue->packets_sent_total = 0;
 578                 data_queue->packets_sent_failed = 0;
 579
 580                 data_queue->packets_received_total = 0;
 581                 data_queue->packets_received_failed = 0;
 582         }
 583 }
 584
 585 static int
 586 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
 587                 const struct rte_cryptodev_qp_conf *qp_conf,
 588                 int socket_id,
 589                 struct rte_mempool *session_pool __rte_unused)
 590 {
 591         int ret;
 592         struct virtqueue *vq;
 593
 594         PMD_INIT_FUNC_TRACE();
 595
 596         /* if virtio dev is started, do not touch the virtqueues */
 597         if (dev->data->dev_started)
 598                 return 0;
 599
 600         ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
 601                         qp_conf->nb_descriptors, socket_id, &vq);
 602         if (ret < 0) {
 603                 VIRTIO_CRYPTO_INIT_LOG_ERR(
 604                         "virtio crypto data queue initialization failed\n");
 605                 return ret;
 606         }
 607
 608         dev->data->queue_pairs[queue_pair_id] = vq;
 609
 610         return 0;
 611 }
 612
 613 static int
 614 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
 615 {
 616         struct virtqueue *vq
 617                 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
 618
 619         PMD_INIT_FUNC_TRACE();
 620
 621         if (vq == NULL) {
 622                 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
 623                 return 0;
 624         }
 625
 626         virtio_crypto_queue_release(vq);
 627         return 0;
 628 }
 629
 630 static int
 631 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
 632 {
 633         uint64_t host_features;
 634
 635         PMD_INIT_FUNC_TRACE();
 636
 637         /* Prepare guest_features: feature that driver wants to support */
 638         VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
 639                 req_features);
 640
 641         /* Read device(host) feature bits */
 642         host_features = VTPCI_OPS(hw)->get_features(hw);
 643         VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
 644                 host_features);
 645
 646         /*
 647          * Negotiate features: Subset of device feature bits are written back
 648          * guest feature bits.
 649          */
 650         hw->guest_features = req_features;
 651         hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
 652                                                         host_features);
 653         VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
 654                 hw->guest_features);
 655
 656         if (hw->modern) {
 657                 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
 658                         VIRTIO_CRYPTO_INIT_LOG_ERR(
 659                                 "VIRTIO_F_VERSION_1 features is not enabled.");
 660                         return -1;
 661                 }
 662                 vtpci_cryptodev_set_status(hw,
 663                         VIRTIO_CONFIG_STATUS_FEATURES_OK);
 664                 if (!(vtpci_cryptodev_get_status(hw) &
 665                         VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
 666                         VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
 667                                                 "status!");
 668                         return -1;
 669                 }
 670         }
 671
 672         hw->req_guest_features = req_features;
 673
 674         return 0;
 675 }
 676
 677 /* reset device and renegotiate features if needed */
 678 static int
 679 virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
 680         uint64_t req_features)
 681 {
 682         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
 683         struct virtio_crypto_config local_config;
 684         struct virtio_crypto_config *config = &local_config;
 685
 686         PMD_INIT_FUNC_TRACE();
 687
 688         /* Reset the device although not necessary at startup */
 689         vtpci_cryptodev_reset(hw);
 690
 691         /* Tell the host we've noticed this device. */
 692         vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
 693
 694         /* Tell the host we've known how to drive the device. */
 695         vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
 696         if (virtio_negotiate_features(hw, req_features) < 0)
 697                 return -1;
 698
 699         /* Get status of the device */
 700         vtpci_read_cryptodev_config(hw,
 701                 offsetof(struct virtio_crypto_config, status),
 702                 &config->status, sizeof(config->status));
 703         if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
 704                 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
 705                                 "not ready");
 706                 return -1;
 707         }
 708
 709         /* Get number of data queues */
 710         vtpci_read_cryptodev_config(hw,
 711                 offsetof(struct virtio_crypto_config, max_dataqueues),
 712                 &config->max_dataqueues,
 713                 sizeof(config->max_dataqueues));
 714         hw->max_dataqueues = config->max_dataqueues;
 715
 716         VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
 717                 hw->max_dataqueues);
 718
 719         return 0;
 720 }
 721
 722 /*
 723  * This function is based on probe() function
 724  * It returns 0 on success.
 725  */
 726 static int
 727 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
 728                 struct rte_cryptodev_pmd_init_params *init_params)
 729 {
 730         struct rte_cryptodev *cryptodev;
 731         struct virtio_crypto_hw *hw;
 732
 733         PMD_INIT_FUNC_TRACE();
 734
 735         cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
 736                                         init_params);
 737         if (cryptodev == NULL)
 738                 return -ENODEV;
 739
 740         cryptodev->driver_id = cryptodev_virtio_driver_id;
 741         cryptodev->dev_ops = &virtio_crypto_dev_ops;
 742
 743         cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
 744         cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
 745
 746         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 747                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
 748
 749         hw = cryptodev->data->dev_private;
 750         hw->dev_id = cryptodev->data->dev_id;
 751         hw->virtio_dev_capabilities = virtio_capabilities;
 752
 753         VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
 754                 cryptodev->data->dev_id, pci_dev->id.vendor_id,
 755                 pci_dev->id.device_id);
 756
 757         /* pci device init */
 758         if (vtpci_cryptodev_init(pci_dev, hw))
 759                 return -1;
 760
 761         if (virtio_crypto_init_device(cryptodev,
 762                         VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
 763                 return -1;
 764
 765         return 0;
 766 }
 767
 768 static int
 769 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
 770 {
 771         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
 772
 773         PMD_INIT_FUNC_TRACE();
 774
 775         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
 776                 return -EPERM;
 777
 778         if (cryptodev->data->dev_started) {
 779                 virtio_crypto_dev_stop(cryptodev);
 780                 virtio_crypto_dev_close(cryptodev);
 781         }
 782
 783         cryptodev->dev_ops = NULL;
 784         cryptodev->enqueue_burst = NULL;
 785         cryptodev->dequeue_burst = NULL;
 786
 787         /* release control queue */
 788         virtio_crypto_queue_release(hw->cvq);
 789
 790         rte_free(cryptodev->data);
 791         cryptodev->data = NULL;
 792
 793         VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
 794
 795         return 0;
 796 }
 797
 798 static int
 799 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
 800         struct rte_cryptodev_config *config __rte_unused)
 801 {
 802         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
 803
 804         PMD_INIT_FUNC_TRACE();
 805
 806         if (virtio_crypto_init_device(cryptodev,
 807                         VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
 808                 return -1;
 809
 810         /* setup control queue
 811          * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
 812          * config->max_dataqueues is the control queue
 813          */
 814         if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
 815                 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
 816                 return -1;
 817         }
 818         virtio_crypto_ctrlq_start(cryptodev);
 819
 820         return 0;
 821 }
 822
 823 static void
 824 virtio_crypto_dev_stop(struct rte_cryptodev *dev)
 825 {
 826         struct virtio_crypto_hw *hw = dev->data->dev_private;
 827
 828         PMD_INIT_FUNC_TRACE();
 829         VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
 830
 831         vtpci_cryptodev_reset(hw);
 832
 833         virtio_crypto_dev_free_mbufs(dev);
 834         virtio_crypto_free_queues(dev);
 835
 836         dev->data->dev_started = 0;
 837 }
 838
 839 static int
 840 virtio_crypto_dev_start(struct rte_cryptodev *dev)
 841 {
 842         struct virtio_crypto_hw *hw = dev->data->dev_private;
 843
 844         if (dev->data->dev_started)
 845                 return 0;
 846
 847         /* Do final configuration before queue engine starts */
 848         virtio_crypto_dataq_start(dev);
 849         vtpci_cryptodev_reinit_complete(hw);
 850
 851         dev->data->dev_started = 1;
 852
 853         return 0;
 854 }
 855
 856 static void
 857 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
 858 {
 859         uint32_t i;
 860         struct virtio_crypto_hw *hw = dev->data->dev_private;
 861
 862         for (i = 0; i < hw->max_dataqueues; i++) {
 863                 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
 864                         "and unused buf", i);
 865                 VIRTQUEUE_DUMP((struct virtqueue *)
 866                         dev->data->queue_pairs[i]);
 867
 868                 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
 869                                 i, dev->data->queue_pairs[i]);
 870
 871                 virtqueue_detatch_unused(dev->data->queue_pairs[i]);
 872
 873                 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
 874                                         "unused buf", i);
 875                 VIRTQUEUE_DUMP(
 876                         (struct virtqueue *)dev->data->queue_pairs[i]);
 877         }
 878 }
 879
 880 static unsigned int
 881 virtio_crypto_sym_get_session_private_size(
 882                 struct rte_cryptodev *dev __rte_unused)
 883 {
 884         PMD_INIT_FUNC_TRACE();
 885
 886         return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
 887 }
 888
 889 static int
 890 virtio_crypto_check_sym_session_paras(
 891                 struct rte_cryptodev *dev)
 892 {
 893         struct virtio_crypto_hw *hw;
 894
 895         PMD_INIT_FUNC_TRACE();
 896
 897         if (unlikely(dev == NULL)) {
 898                 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
 899                 return -1;
 900         }
 901         if (unlikely(dev->data == NULL)) {
 902                 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
 903                 return -1;
 904         }
 905         hw = dev->data->dev_private;
 906         if (unlikely(hw == NULL)) {
 907                 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
 908                 return -1;
 909         }
 910         if (unlikely(hw->cvq == NULL)) {
 911                 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
 912                 return -1;
 913         }
 914
 915         return 0;
 916 }
 917
 918 static int
 919 virtio_crypto_check_sym_clear_session_paras(
 920                 struct rte_cryptodev *dev,
 921                 struct rte_cryptodev_sym_session *sess)
 922 {
 923         PMD_INIT_FUNC_TRACE();
 924
 925         if (sess == NULL) {
 926                 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
 927                 return -1;
 928         }
 929
 930         return virtio_crypto_check_sym_session_paras(dev);
 931 }
 932
 933 #define NUM_ENTRY_SYM_CLEAR_SESSION 2
 934
 935 static void
 936 virtio_crypto_sym_clear_session(
 937                 struct rte_cryptodev *dev,
 938                 struct rte_cryptodev_sym_session *sess)
 939 {
 940         struct virtio_crypto_hw *hw;
 941         struct virtqueue *vq;
 942         struct virtio_crypto_session *session;
 943         struct virtio_crypto_op_ctrl_req *ctrl;
 944         struct vring_desc *desc;
 945         uint8_t *status;
 946         uint8_t needed = 1;
 947         uint32_t head;
 948         uint8_t *malloc_virt_addr;
 949         uint64_t malloc_phys_addr;
 950         uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
 951         uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
 952         uint32_t desc_offset = len_op_ctrl_req + len_inhdr;
 953
 954         PMD_INIT_FUNC_TRACE();
 955
 956         if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
 957                 return;
 958
 959         hw = dev->data->dev_private;
 960         vq = hw->cvq;
 961         session = (struct virtio_crypto_session *)get_sym_session_private_data(
 962                 sess, cryptodev_virtio_driver_id);
 963         if (session == NULL) {
 964                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
 965                 return;
 966         }
 967
 968         VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
 969                         "vq = %p", vq->vq_desc_head_idx, vq);
 970
 971         if (vq->vq_free_cnt < needed) {
 972                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
 973                                 "vq->vq_free_cnt = %d is less than %d, "
 974                                 "not enough", vq->vq_free_cnt, needed);
 975                 return;
 976         }
 977
 978         /*
 979          * malloc memory to store information of ctrl request op,
 980          * returned status and desc vring
 981          */
 982         malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
 983                 + NUM_ENTRY_SYM_CLEAR_SESSION
 984                 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
 985         if (malloc_virt_addr == NULL) {
 986                 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
 987                 return;
 988         }
 989         malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);
 990
 991         /* assign ctrl request op part */
 992         ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
 993         ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
 994         /* default data virtqueue is 0 */
 995         ctrl->header.queue_id = 0;
 996         ctrl->u.destroy_session.session_id = session->session_id;
 997
 998         /* status part */
 999         status = &(((struct virtio_crypto_inhdr *)
1000                 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
1001         *status = VIRTIO_CRYPTO_ERR;
1002
1003         /* indirect desc vring part */
1004         desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
1005                 + desc_offset);
1006
1007         /* ctrl request part */
1008         desc[0].addr = malloc_phys_addr;
1009         desc[0].len = len_op_ctrl_req;
1010         desc[0].flags = VRING_DESC_F_NEXT;
1011         desc[0].next = 1;
1012
1013         /* status part */
1014         desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
1015         desc[1].len = len_inhdr;
1016         desc[1].flags = VRING_DESC_F_WRITE;
1017
1018         /* use only a single desc entry */
1019         head = vq->vq_desc_head_idx;
1020         vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
1021         vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
1022         vq->vq_ring.desc[head].len
1023                 = NUM_ENTRY_SYM_CLEAR_SESSION
1024                 * sizeof(struct vring_desc);
1025         vq->vq_free_cnt -= needed;
1026
1027         vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
1028
1029         vq_update_avail_ring(vq, head);
1030         vq_update_avail_idx(vq);
1031
1032         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
1033                                         vq->vq_queue_index);
1034
1035         virtqueue_notify(vq);
1036
1037         rte_rmb();
1038         while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
1039                 rte_rmb();
1040                 usleep(100);
1041         }
1042
1043         while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
1044                 uint32_t idx, desc_idx, used_idx;
1045                 struct vring_used_elem *uep;
1046
1047                 used_idx = (uint32_t)(vq->vq_used_cons_idx
1048                                 & (vq->vq_nentries - 1));
1049                 uep = &vq->vq_ring.used->ring[used_idx];
1050                 idx = (uint32_t) uep->id;
1051                 desc_idx = idx;
1052                 while (vq->vq_ring.desc[desc_idx].flags
1053                                 & VRING_DESC_F_NEXT) {
1054                         desc_idx = vq->vq_ring.desc[desc_idx].next;
1055                         vq->vq_free_cnt++;
1056                 }
1057
1058                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
1059                 vq->vq_desc_head_idx = idx;
1060                 vq->vq_used_cons_idx++;
1061                 vq->vq_free_cnt++;
1062         }
1063
1064         if (*status != VIRTIO_CRYPTO_OK) {
1065                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
1066                                 "status=%"PRIu32", session_id=%"PRIu64"",
1067                                 *status, session->session_id);
1068                 rte_free(malloc_virt_addr);
1069                 return;
1070         }
1071
1072         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
1073                         "vq->vq_desc_head_idx=%d",
1074                         vq->vq_free_cnt, vq->vq_desc_head_idx);
1075
1076         VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
1077                         session->session_id);
1078
1079         memset(session, 0, sizeof(struct virtio_crypto_session));
1080         struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
1081         set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
1082         rte_mempool_put(sess_mp, session);
1083         rte_free(malloc_virt_addr);
1084 }
1085
1086 static struct rte_crypto_cipher_xform *
1087 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
1088 {
1089         do {
1090                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1091                         return &xform->cipher;
1092
1093                 xform = xform->next;
1094         } while (xform);
1095
1096         return NULL;
1097 }
1098
1099 static struct rte_crypto_auth_xform *
1100 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
1101 {
1102         do {
1103                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1104                         return &xform->auth;
1105
1106                 xform = xform->next;
1107         } while (xform);
1108
1109         return NULL;
1110 }
1111
1112 /** Get xform chain order */
1113 static int
1114 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
1115 {
1116         if (xform == NULL)
1117                 return -1;
1118
1119         /* Cipher Only */
1120         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1121                         xform->next == NULL)
1122                 return VIRTIO_CRYPTO_CMD_CIPHER;
1123
1124         /* Authentication Only */
1125         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1126                         xform->next == NULL)
1127                 return VIRTIO_CRYPTO_CMD_AUTH;
1128
1129         /* Authenticate then Cipher */
1130         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1131                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1132                 return VIRTIO_CRYPTO_CMD_HASH_CIPHER;
1133
1134         /* Cipher then Authenticate */
1135         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1136                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1137                 return VIRTIO_CRYPTO_CMD_CIPHER_HASH;
1138
1139         return -1;
1140 }
1141
1142 static int
1143 virtio_crypto_sym_pad_cipher_param(
1144                 struct virtio_crypto_cipher_session_para *para,
1145                 struct rte_crypto_cipher_xform *cipher_xform)
1146 {
1147         switch (cipher_xform->algo) {
1148         case RTE_CRYPTO_CIPHER_AES_CBC:
1149                 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
1150                 break;
1151         default:
1152                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
1153                                 "Cipher alg %u", cipher_xform->algo);
1154                 return -1;
1155         }
1156
1157         para->keylen = cipher_xform->key.length;
1158         switch (cipher_xform->op) {
1159         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
1160                 para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
1161                 break;
1162         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
1163                 para->op = VIRTIO_CRYPTO_OP_DECRYPT;
1164                 break;
1165         default:
1166                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
1167                                         "parameter");
1168                 return -1;
1169         }
1170
1171         return 0;
1172 }
1173
1174 static int
1175 virtio_crypto_sym_pad_auth_param(
1176                 struct virtio_crypto_op_ctrl_req *ctrl,
1177                 struct rte_crypto_auth_xform *auth_xform)
1178 {
1179         uint32_t *algo;
1180         struct virtio_crypto_alg_chain_session_para *para =
1181                 &(ctrl->u.sym_create_session.u.chain.para);
1182
1183         switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
1184         case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
1185                 algo = &(para->u.hash_param.algo);
1186                 break;
1187         case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
1188                 algo = &(para->u.mac_param.algo);
1189                 break;
1190         default:
1191                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
1192                         "specified",
1193                         ctrl->u.sym_create_session.u.chain.para.hash_mode);
1194                 return -1;
1195         }
1196
1197         switch (auth_xform->algo) {
1198         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1199                 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
1200                 break;
1201         default:
1202                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1203                         "Crypto: Undefined Hash algo %u specified",
1204                         auth_xform->algo);
1205                 return -1;
1206         }
1207
1208         return 0;
1209 }
1210
1211 static int
1212 virtio_crypto_sym_pad_op_ctrl_req(
1213                 struct virtio_crypto_op_ctrl_req *ctrl,
1214                 struct rte_crypto_sym_xform *xform, bool is_chainned,
1215                 uint8_t **cipher_key_data, uint8_t **auth_key_data,
1216                 struct virtio_crypto_session *session)
1217 {
1218         int ret;
1219         struct rte_crypto_auth_xform *auth_xform = NULL;
1220         struct rte_crypto_cipher_xform *cipher_xform = NULL;
1221
1222         /* Get cipher xform from crypto xform chain */
1223         cipher_xform = virtio_crypto_get_cipher_xform(xform);
1224         if (cipher_xform) {
1225                 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
1226                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1227                                 "cipher IV size cannot be longer than %u",
1228                                 VIRTIO_CRYPTO_MAX_IV_SIZE);
1229                         return -1;
1230                 }
1231                 if (is_chainned)
1232                         ret = virtio_crypto_sym_pad_cipher_param(
1233                                 &ctrl->u.sym_create_session.u.chain.para
1234                                                 .cipher_param, cipher_xform);
1235                 else
1236                         ret = virtio_crypto_sym_pad_cipher_param(
1237                                 &ctrl->u.sym_create_session.u.cipher.para,
1238                                 cipher_xform);
1239
1240                 if (ret < 0) {
1241                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1242                                 "pad cipher parameter failed");
1243                         return -1;
1244                 }
1245
1246                 *cipher_key_data = cipher_xform->key.data;
1247
1248                 session->iv.offset = cipher_xform->iv.offset;
1249                 session->iv.length = cipher_xform->iv.length;
1250         }
1251
1252         /* Get auth xform from crypto xform chain */
1253         auth_xform = virtio_crypto_get_auth_xform(xform);
1254         if (auth_xform) {
1255                 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
1256                 struct virtio_crypto_alg_chain_session_para *para =
1257                         &(ctrl->u.sym_create_session.u.chain.para);
1258                 if (auth_xform->key.length) {
1259                         para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
1260                         para->u.mac_param.auth_key_len =
1261                                 (uint32_t)auth_xform->key.length;
1262                         para->u.mac_param.hash_result_len =
1263                                 auth_xform->digest_length;
1264
1265                         *auth_key_data = auth_xform->key.data;
1266                 } else {
1267                         para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
1268                         para->u.hash_param.hash_result_len =
1269                                 auth_xform->digest_length;
1270                 }
1271
1272                 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
1273                 if (ret < 0) {
1274                         VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
1275                                                 "failed");
1276                         return -1;
1277                 }
1278         }
1279
1280         return 0;
1281 }
1282
1283 static int
1284 virtio_crypto_check_sym_configure_session_paras(
1285                 struct rte_cryptodev *dev,
1286                 struct rte_crypto_sym_xform *xform,
1287                 struct rte_cryptodev_sym_session *sym_sess,
1288                 struct rte_mempool *mempool)
1289 {
1290         if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
1291                 unlikely(mempool == NULL)) {
1292                 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
1293                 return -1;
1294         }
1295
1296         if (virtio_crypto_check_sym_session_paras(dev) < 0)
1297                 return -1;
1298
1299         return 0;
1300 }
1301
1302 static int
1303 virtio_crypto_sym_configure_session(
1304                 struct rte_cryptodev *dev,
1305                 struct rte_crypto_sym_xform *xform,
1306                 struct rte_cryptodev_sym_session *sess,
1307                 struct rte_mempool *mempool)
1308 {
1309         int ret;
1310         struct virtio_crypto_session crypto_sess;
1311         void *session_private = &crypto_sess;
1312         struct virtio_crypto_session *session;
1313         struct virtio_crypto_op_ctrl_req *ctrl_req;
1314         enum virtio_crypto_cmd_id cmd_id;
1315         uint8_t *cipher_key_data = NULL;
1316         uint8_t *auth_key_data = NULL;
1317         struct virtio_crypto_hw *hw;
1318         struct virtqueue *control_vq;
1319
1320         PMD_INIT_FUNC_TRACE();
1321
1322         ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
1323                         sess, mempool);
1324         if (ret < 0) {
1325                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
1326                 return ret;
1327         }
1328
1329         if (rte_mempool_get(mempool, &session_private)) {
1330                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1331                         "Couldn't get object from session mempool");
1332                 return -ENOMEM;
1333         }
1334
1335         session = (struct virtio_crypto_session *)session_private;
1336         memset(session, 0, sizeof(struct virtio_crypto_session));
1337         ctrl_req = &session->ctrl;
1338         ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
1339         /* FIXME: support multiqueue */
1340         ctrl_req->header.queue_id = 0;
1341
1342         hw = dev->data->dev_private;
1343         control_vq = hw->cvq;
1344
1345         cmd_id = virtio_crypto_get_chain_order(xform);
1346         if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
1347                 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1348                         = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
1349         if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
1350                 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1351                         = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1352
1353         switch (cmd_id) {
1354         case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
1355         case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
1356                 ctrl_req->u.sym_create_session.op_type
1357                         = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;
1358
1359                 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
1360                         xform, true, &cipher_key_data, &auth_key_data, session);
1361                 if (ret < 0) {
1362                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1363                                 "padding sym op ctrl req failed");
1364                         goto error_out;
1365                 }
1366                 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1367                         cipher_key_data, auth_key_data, session);
1368                 if (ret < 0) {
1369                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1370                                 "create session failed: %d", ret);
1371                         goto error_out;
1372                 }
1373                 break;
1374         case VIRTIO_CRYPTO_CMD_CIPHER:
1375                 ctrl_req->u.sym_create_session.op_type
1376                         = VIRTIO_CRYPTO_SYM_OP_CIPHER;
1377                 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
1378                         false, &cipher_key_data, &auth_key_data, session);
1379                 if (ret < 0) {
1380                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1381                                 "padding sym op ctrl req failed");
1382                         goto error_out;
1383                 }
1384                 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1385                         cipher_key_data, NULL, session);
1386                 if (ret < 0) {
1387                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1388                                 "create session failed: %d", ret);
1389                         goto error_out;
1390                 }
1391                 break;
1392         default:
1393                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1394                         "Unsupported operation chain order parameter");
1395                 goto error_out;
1396         }
1397
1398         set_sym_session_private_data(sess, dev->driver_id,
1399                 session_private);
1400
1401         return 0;
1402
1403 error_out:
1404         return -1;
1405 }
1406
1407 static void
1408 virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
1409                 struct rte_cryptodev_info *info)
1410 {
1411         struct virtio_crypto_hw *hw = dev->data->dev_private;
1412
1413         PMD_INIT_FUNC_TRACE();
1414
1415         if (info != NULL) {
1416                 info->driver_id = cryptodev_virtio_driver_id;
1417                 info->feature_flags = dev->feature_flags;
1418                 info->max_nb_queue_pairs = hw->max_dataqueues;
1419                 /* No limit of number of sessions */
1420                 info->sym.max_nb_sessions = 0;
1421                 info->capabilities = hw->virtio_dev_capabilities;
1422         }
1423 }
1424
1425 static int
1426 crypto_virtio_pci_probe(
1427         struct rte_pci_driver *pci_drv __rte_unused,
1428         struct rte_pci_device *pci_dev)
1429 {
1430         struct rte_cryptodev_pmd_init_params init_params = {
1431                 .name = "",
1432                 .socket_id = rte_socket_id(),
1433                 .private_data_size = sizeof(struct virtio_crypto_hw)
1434         };
1435         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1436
1437         VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
1438                         pci_dev->addr.bus,
1439                         pci_dev->addr.devid,
1440                         pci_dev->addr.function);
1441
1442         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
1443
1444         return crypto_virtio_create(name, pci_dev, &init_params);
1445 }
1446
1447 static int
1448 crypto_virtio_pci_remove(
1449         struct rte_pci_device *pci_dev __rte_unused)
1450 {
1451         struct rte_cryptodev *cryptodev;
1452         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1453
1454         if (pci_dev == NULL)
1455                 return -EINVAL;
1456
1457         rte_pci_device_name(&pci_dev->addr, cryptodev_name,
1458                         sizeof(cryptodev_name));
1459
1460         cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
1461         if (cryptodev == NULL)
1462                 return -ENODEV;
1463
1464         return virtio_crypto_dev_uninit(cryptodev);
1465 }
1466
1467 static struct rte_pci_driver rte_virtio_crypto_driver = {
1468         .id_table = pci_id_virtio_crypto_map,
1469         .drv_flags = 0,
1470         .probe = crypto_virtio_pci_probe,
1471         .remove = crypto_virtio_pci_remove
1472 };
1473
1474 static struct cryptodev_driver virtio_crypto_drv;
1475
1476 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
1477 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
1478         rte_virtio_crypto_driver.driver,
1479         cryptodev_virtio_driver_id);
1480
1481 RTE_INIT(virtio_crypto_init_log)
1482 {
1483         virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init");
1484         if (virtio_crypto_logtype_init >= 0)
1485                 rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE);
1486
1487         virtio_crypto_logtype_session =
1488                 rte_log_register("pmd.crypto.virtio.session");
1489         if (virtio_crypto_logtype_session >= 0)
1490                 rte_log_set_level(virtio_crypto_logtype_session,
1491                                 RTE_LOG_NOTICE);
1492
1493         virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx");
1494         if (virtio_crypto_logtype_rx >= 0)
1495                 rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE);
1496
1497         virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx");
1498         if (virtio_crypto_logtype_tx >= 0)
1499                 rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE);
1500
1501         virtio_crypto_logtype_driver =
1502                 rte_log_register("pmd.crypto.virtio.driver");
1503         if (virtio_crypto_logtype_driver >= 0)
1504                 rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE);
1505 }