drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_atomic.h>
  12 #include <rte_prefetch.h>
  13
  14 #include "qat_logs.h"
  15 #include "qat_device.h"
  16 #include "qat_qp.h"
  17 #include "qat_sym.h"
  18 #include "qat_comp.h"
  19 #include "adf_transport_access_macros.h"
  20
  21
  22 #define ADF_MAX_DESC                            4096
  23 #define ADF_MIN_DESC                            128
  24
  25 #define ADF_ARB_REG_SLOT                        0x1000
  26 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  27
  28 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  29         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  30         (ADF_ARB_REG_SLOT * index), value)
  31
  32 __extension__
  33 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  34                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  35         /* queue pairs which provide an asymmetric crypto service */
  36         [QAT_SERVICE_ASYMMETRIC] = {
  37                 {
  38                         .service_type = QAT_SERVICE_ASYMMETRIC,
  39                         .hw_bundle_num = 0,
  40                         .tx_ring_num = 0,
  41                         .rx_ring_num = 8,
  42                         .tx_msg_size = 64,
  43                         .rx_msg_size = 32,
  44
  45                 }, {
  46                         .service_type = QAT_SERVICE_ASYMMETRIC,
  47                         .hw_bundle_num = 0,
  48                         .tx_ring_num = 1,
  49                         .rx_ring_num = 9,
  50                         .tx_msg_size = 64,
  51                         .rx_msg_size = 32,
  52                 }
  53         },
  54         /* queue pairs which provide a symmetric crypto service */
  55         [QAT_SERVICE_SYMMETRIC] = {
  56                 {
  57                         .service_type = QAT_SERVICE_SYMMETRIC,
  58                         .hw_bundle_num = 0,
  59                         .tx_ring_num = 2,
  60                         .rx_ring_num = 10,
  61                         .tx_msg_size = 128,
  62                         .rx_msg_size = 32,
  63                 },
  64                 {
  65                         .service_type = QAT_SERVICE_SYMMETRIC,
  66                         .hw_bundle_num = 0,
  67                         .tx_ring_num = 3,
  68                         .rx_ring_num = 11,
  69                         .tx_msg_size = 128,
  70                         .rx_msg_size = 32,
  71                 }
  72         },
  73         /* queue pairs which provide a compression service */
  74         [QAT_SERVICE_COMPRESSION] = {
  75                 {
  76                         .service_type = QAT_SERVICE_COMPRESSION,
  77                         .hw_bundle_num = 0,
  78                         .tx_ring_num = 6,
  79                         .rx_ring_num = 14,
  80                         .tx_msg_size = 128,
  81                         .rx_msg_size = 32,
  82                 }, {
  83                         .service_type = QAT_SERVICE_COMPRESSION,
  84                         .hw_bundle_num = 0,
  85                         .tx_ring_num = 7,
  86                         .rx_ring_num = 15,
  87                         .tx_msg_size = 128,
  88                         .rx_msg_size = 32,
  89                 }
  90         }
  91 };
  92
  93 __extension__
  94 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
  95                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  96         /* queue pairs which provide an asymmetric crypto service */
  97         [QAT_SERVICE_ASYMMETRIC] = {
  98                 {
  99                         .service_type = QAT_SERVICE_ASYMMETRIC,
 100                         .hw_bundle_num = 0,
 101                         .tx_ring_num = 0,
 102                         .rx_ring_num = 4,
 103                         .tx_msg_size = 64,
 104                         .rx_msg_size = 32,
 105                 }
 106         },
 107         /* queue pairs which provide a symmetric crypto service */
 108         [QAT_SERVICE_SYMMETRIC] = {
 109                 {
 110                         .service_type = QAT_SERVICE_SYMMETRIC,
 111                         .hw_bundle_num = 0,
 112                         .tx_ring_num = 1,
 113                         .rx_ring_num = 5,
 114                         .tx_msg_size = 128,
 115                         .rx_msg_size = 32,
 116                 }
 117         },
 118         /* queue pairs which provide a compression service */
 119         [QAT_SERVICE_COMPRESSION] = {
 120                 {
 121                         .service_type = QAT_SERVICE_COMPRESSION,
 122                         .hw_bundle_num = 0,
 123                         .tx_ring_num = 3,
 124                         .rx_ring_num = 7,
 125                         .tx_msg_size = 128,
 126                         .rx_msg_size = 32,
 127                 }
 128         }
 129 };
 130
 131 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 132         uint32_t queue_size_bytes);
 133 static void qat_queue_delete(struct qat_queue *queue);
 134 static int qat_queue_create(struct qat_pci_device *qat_dev,
 135         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 136 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 137         uint32_t *queue_size_for_csr);
 138 static void adf_configure_queues(struct qat_qp *queue);
 139 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 140         rte_spinlock_t *lock);
 141 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 142         rte_spinlock_t *lock);
 143
 144
 145 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
 146                 enum qat_service_type service)
 147 {
 148         int i, count;
 149
 150         for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
 151                 if (qp_hw_data[i].service_type == service)
 152                         count++;
 153         return count;
 154 }
 155
 156 static const struct rte_memzone *
 157 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 158                         int socket_id)
 159 {
 160         const struct rte_memzone *mz;
 161
 162         mz = rte_memzone_lookup(queue_name);
 163         if (mz != 0) {
 164                 if (((size_t)queue_size <= mz->len) &&
 165                                 ((socket_id == SOCKET_ID_ANY) ||
 166                                         (socket_id == mz->socket_id))) {
 167                         QAT_LOG(DEBUG, "re-use memzone already "
 168                                         "allocated for %s", queue_name);
 169                         return mz;
 170                 }
 171
 172                 QAT_LOG(ERR, "Incompatible memzone already "
 173                                 "allocated %s, size %u, socket %d. "
 174                                 "Requested size %u, socket %u",
 175                                 queue_name, (uint32_t)mz->len,
 176                                 mz->socket_id, queue_size, socket_id);
 177                 return NULL;
 178         }
 179
 180         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 181                                         queue_name, queue_size, socket_id);
 182         return rte_memzone_reserve_aligned(queue_name, queue_size,
 183                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 184 }
 185
 186 int qat_qp_setup(struct qat_pci_device *qat_dev,
 187                 struct qat_qp **qp_addr,
 188                 uint16_t queue_pair_id,
 189                 struct qat_qp_config *qat_qp_conf)
 190
 191 {
 192         struct qat_qp *qp;
 193         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 194         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 195         uint32_t i;
 196
 197         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 198                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 199
 200         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 201                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 202                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 203                                 qat_qp_conf->nb_descriptors);
 204                 return -EINVAL;
 205         }
 206
 207         if (pci_dev->mem_resource[0].addr == NULL) {
 208                 QAT_LOG(ERR, "Could not find VF config space "
 209                                 "(UIO driver attached?).");
 210                 return -EINVAL;
 211         }
 212
 213         /* Allocate the queue pair data structure. */
 214         qp = rte_zmalloc("qat PMD qp metadata",
 215                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
 216         if (qp == NULL) {
 217                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 218                 return -ENOMEM;
 219         }
 220         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 221         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
 222                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 223                         RTE_CACHE_LINE_SIZE);
 224         if (qp->op_cookies == NULL) {
 225                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 226                 rte_free(qp);
 227                 return -ENOMEM;
 228         }
 229
 230         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 231         qp->inflights16 = 0;
 232
 233         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 234                                         ADF_RING_DIR_TX) != 0) {
 235                 QAT_LOG(ERR, "Tx queue create failed "
 236                                 "queue_pair_id=%u", queue_pair_id);
 237                 goto create_err;
 238         }
 239
 240         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 241                                         ADF_RING_DIR_RX) != 0) {
 242                 QAT_LOG(ERR, "Rx queue create failed "
 243                                 "queue_pair_id=%hu", queue_pair_id);
 244                 qat_queue_delete(&(qp->tx_q));
 245                 goto create_err;
 246         }
 247
 248         adf_configure_queues(qp);
 249         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
 250                                         &qat_dev->arb_csr_lock);
 251
 252         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 253                                         "%s%d_cookies_%s_qp%hu",
 254                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 255                 qat_qp_conf->service_str, queue_pair_id);
 256
 257         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 258         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 259         if (qp->op_cookie_pool == NULL)
 260                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 261                                 qp->nb_descriptors,
 262                                 qat_qp_conf->cookie_size, 64, 0,
 263                                 NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
 264                                 0);
 265         if (!qp->op_cookie_pool) {
 266                 QAT_LOG(ERR, "QAT PMD Cannot create"
 267                                 " op mempool");
 268                 goto create_err;
 269         }
 270
 271         for (i = 0; i < qp->nb_descriptors; i++) {
 272                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 273                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 274                         goto create_err;
 275                 }
 276         }
 277
 278         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 279         qp->build_request = qat_qp_conf->build_request;
 280         qp->service_type = qat_qp_conf->hw->service_type;
 281         qp->qat_dev = qat_dev;
 282
 283         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 284                         queue_pair_id, op_cookie_pool_name);
 285
 286         *qp_addr = qp;
 287         return 0;
 288
 289 create_err:
 290         if (qp->op_cookie_pool)
 291                 rte_mempool_free(qp->op_cookie_pool);
 292         rte_free(qp->op_cookies);
 293         rte_free(qp);
 294         return -EFAULT;
 295 }
 296
 297 int qat_qp_release(struct qat_qp **qp_addr)
 298 {
 299         struct qat_qp *qp = *qp_addr;
 300         uint32_t i;
 301
 302         if (qp == NULL) {
 303                 QAT_LOG(DEBUG, "qp already freed");
 304                 return 0;
 305         }
 306
 307         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 308                                 qp->qat_dev->qat_dev_id);
 309
 310         /* Don't free memory if there are still responses to be processed */
 311         if (qp->inflights16 == 0) {
 312                 qat_queue_delete(&(qp->tx_q));
 313                 qat_queue_delete(&(qp->rx_q));
 314         } else {
 315                 return -EAGAIN;
 316         }
 317
 318         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
 319                                         &qp->qat_dev->arb_csr_lock);
 320
 321         for (i = 0; i < qp->nb_descriptors; i++)
 322                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 323
 324         if (qp->op_cookie_pool)
 325                 rte_mempool_free(qp->op_cookie_pool);
 326
 327         rte_free(qp->op_cookies);
 328         rte_free(qp);
 329         *qp_addr = NULL;
 330         return 0;
 331 }
 332
 333
 334 static void qat_queue_delete(struct qat_queue *queue)
 335 {
 336         const struct rte_memzone *mz;
 337         int status = 0;
 338
 339         if (queue == NULL) {
 340                 QAT_LOG(DEBUG, "Invalid queue");
 341                 return;
 342         }
 343         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 344                         queue->hw_queue_number, queue->memz_name);
 345
 346         mz = rte_memzone_lookup(queue->memz_name);
 347         if (mz != NULL) {
 348                 /* Write an unused pattern to the queue memory. */
 349                 memset(queue->base_addr, 0x7F, queue->queue_size);
 350                 status = rte_memzone_free(mz);
 351                 if (status != 0)
 352                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 353                                         status, queue->memz_name);
 354         } else {
 355                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 356                                 queue->memz_name);
 357         }
 358 }
 359
 360 static int
 361 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 362                 struct qat_qp_config *qp_conf, uint8_t dir)
 363 {
 364         uint64_t queue_base;
 365         void *io_addr;
 366         const struct rte_memzone *qp_mz;
 367         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 368         int ret = 0;
 369         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 370                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 371         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 372
 373         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 374         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 375                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 376
 377         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 378                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 379                 return -EINVAL;
 380         }
 381
 382         /*
 383          * Allocate a memzone for the queue - create a unique name.
 384          */
 385         snprintf(queue->memz_name, sizeof(queue->memz_name),
 386                         "%s_%d_%s_%s_%d_%d",
 387                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 388                 qp_conf->service_str, "qp_mem",
 389                 queue->hw_bundle_number, queue->hw_queue_number);
 390         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 391                         qp_conf->socket_id);
 392         if (qp_mz == NULL) {
 393                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 394                 return -ENOMEM;
 395         }
 396
 397         queue->base_addr = (char *)qp_mz->addr;
 398         queue->base_phys_addr = qp_mz->iova;
 399         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 400                         queue_size_bytes)) {
 401                 QAT_LOG(ERR, "Invalid alignment on queue create "
 402                                         " 0x%"PRIx64"\n",
 403                                         queue->base_phys_addr);
 404                 ret = -EFAULT;
 405                 goto queue_create_err;
 406         }
 407
 408         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 409                         &(queue->queue_size)) != 0) {
 410                 QAT_LOG(ERR, "Invalid num inflights");
 411                 ret = -EINVAL;
 412                 goto queue_create_err;
 413         }
 414
 415         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 416                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
 417         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 418
 419         if (queue->max_inflights < 2) {
 420                 QAT_LOG(ERR, "Invalid num inflights");
 421                 ret = -EINVAL;
 422                 goto queue_create_err;
 423         }
 424         queue->head = 0;
 425         queue->tail = 0;
 426         queue->msg_size = desc_size;
 427
 428         /*
 429          * Write an unused pattern to the queue memory.
 430          */
 431         memset(queue->base_addr, 0x7F, queue_size_bytes);
 432
 433         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 434                                         queue->queue_size);
 435
 436         io_addr = pci_dev->mem_resource[0].addr;
 437
 438         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 439                         queue->hw_queue_number, queue_base);
 440
 441         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 442                 " nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
 443                         queue->memz_name,
 444                         queue->queue_size, queue_size_bytes,
 445                         qp_conf->nb_descriptors, desc_size,
 446                         queue->max_inflights, queue->modulo_mask);
 447
 448         return 0;
 449
 450 queue_create_err:
 451         rte_memzone_free(qp_mz);
 452         return ret;
 453 }
 454
 455 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 456                                         uint32_t queue_size_bytes)
 457 {
 458         if (((queue_size_bytes - 1) & phys_addr) != 0)
 459                 return -EINVAL;
 460         return 0;
 461 }
 462
 463 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 464         uint32_t *p_queue_size_for_csr)
 465 {
 466         uint8_t i = ADF_MIN_RING_SIZE;
 467
 468         for (; i <= ADF_MAX_RING_SIZE; i++)
 469                 if ((msg_size * msg_num) ==
 470                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 471                         *p_queue_size_for_csr = i;
 472                         return 0;
 473                 }
 474         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 475         return -EINVAL;
 476 }
 477
 478 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 479                                         rte_spinlock_t *lock)
 480 {
 481         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 482                                         (ADF_ARB_REG_SLOT *
 483                                                         txq->hw_bundle_number);
 484         uint32_t value;
 485
 486         rte_spinlock_lock(lock);
 487         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 488         value |= (0x01 << txq->hw_queue_number);
 489         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 490         rte_spinlock_unlock(lock);
 491 }
 492
 493 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 494                                         rte_spinlock_t *lock)
 495 {
 496         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 497                                         (ADF_ARB_REG_SLOT *
 498                                                         txq->hw_bundle_number);
 499         uint32_t value;
 500
 501         rte_spinlock_lock(lock);
 502         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 503         value &= ~(0x01 << txq->hw_queue_number);
 504         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 505         rte_spinlock_unlock(lock);
 506 }
 507
 508 static void adf_configure_queues(struct qat_qp *qp)
 509 {
 510         uint32_t queue_config;
 511         struct qat_queue *queue = &qp->tx_q;
 512
 513         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 514
 515         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 516                         queue->hw_queue_number, queue_config);
 517
 518         queue = &qp->rx_q;
 519         queue_config =
 520                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 521                                         ADF_RING_NEAR_WATERMARK_512,
 522                                         ADF_RING_NEAR_WATERMARK_0);
 523
 524         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 525                         queue->hw_queue_number, queue_config);
 526 }
 527
 528 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 529 {
 530         return data & modulo_mask;
 531 }
 532
 533 static inline void
 534 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 535         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 536                         q->hw_queue_number, q->tail);
 537         q->nb_pending_requests = 0;
 538         q->csr_tail = q->tail;
 539 }
 540
 541 static inline
 542 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 543 {
 544         uint32_t old_head, new_head;
 545         uint32_t max_head;
 546
 547         old_head = q->csr_head;
 548         new_head = q->head;
 549         max_head = qp->nb_descriptors * q->msg_size;
 550
 551         /* write out free descriptors */
 552         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 553
 554         if (new_head < old_head) {
 555                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 556                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 557         } else {
 558                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 559         }
 560         q->nb_processed_responses = 0;
 561         q->csr_head = new_head;
 562
 563         /* write current head to CSR */
 564         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 565                             q->hw_queue_number, new_head);
 566 }
 567
 568 uint16_t
 569 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 570 {
 571         register struct qat_queue *queue;
 572         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 573         register uint32_t nb_ops_sent = 0;
 574         register int ret;
 575         uint16_t nb_ops_possible = nb_ops;
 576         register uint8_t *base_addr;
 577         register uint32_t tail;
 578         int overflow;
 579
 580         if (unlikely(nb_ops == 0))
 581                 return 0;
 582
 583         /* read params used a lot in main loop into registers */
 584         queue = &(tmp_qp->tx_q);
 585         base_addr = (uint8_t *)queue->base_addr;
 586         tail = queue->tail;
 587
 588         /* Find how many can actually fit on the ring */
 589         tmp_qp->inflights16 += nb_ops;
 590         overflow = tmp_qp->inflights16 - queue->max_inflights;
 591         if (overflow > 0) {
 592                 tmp_qp->inflights16 -= overflow;
 593                 nb_ops_possible = nb_ops - overflow;
 594                 if (nb_ops_possible == 0)
 595                         return 0;
 596         }
 597
 598         while (nb_ops_sent != nb_ops_possible) {
 599                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 600                                 tmp_qp->op_cookies[tail / queue->msg_size],
 601                                 tmp_qp->qat_dev_gen);
 602                 if (ret != 0) {
 603                         tmp_qp->stats.enqueue_err_count++;
 604                         /*
 605                          * This message cannot be enqueued,
 606                          * decrease number of ops that wasn't sent
 607                          */
 608                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
 609                         if (nb_ops_sent == 0)
 610                                 return 0;
 611                         goto kick_tail;
 612                 }
 613
 614                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 615                 ops++;
 616                 nb_ops_sent++;
 617         }
 618 kick_tail:
 619         queue->tail = tail;
 620         tmp_qp->stats.enqueued_count += nb_ops_sent;
 621         queue->nb_pending_requests += nb_ops_sent;
 622         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
 623                     queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
 624                 txq_write_tail(tmp_qp, queue);
 625         }
 626         return nb_ops_sent;
 627 }
 628
 629 uint16_t
 630 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 631 {
 632         struct qat_queue *rx_queue, *tx_queue;
 633         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 634         uint32_t head;
 635         uint32_t resp_counter = 0;
 636         uint8_t *resp_msg;
 637         uint8_t hdr_flags;
 638
 639         rx_queue = &(tmp_qp->rx_q);
 640         tx_queue = &(tmp_qp->tx_q);
 641         head = rx_queue->head;
 642         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 643         hdr_flags = ((struct icp_qat_fw_comn_resp_hdr *)resp_msg)->hdr_flags;
 644
 645         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 646                         resp_counter != nb_ops) {
 647
 648                 if (unlikely(!ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags))) {
 649                         /* Fatal firmware error */
 650                         QAT_LOG(ERR, "QAT Firmware returned invalid response");
 651                         return 0;
 652                 }
 653
 654                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 655                         qat_sym_process_response(ops, resp_msg);
 656                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 657                         qat_comp_process_response(ops, resp_msg);
 658
 659                 head = adf_modulo(head + rx_queue->msg_size,
 660                                   rx_queue->modulo_mask);
 661
 662                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 663                 ops++;
 664                 resp_counter++;
 665         }
 666         if (resp_counter > 0) {
 667                 rx_queue->head = head;
 668                 tmp_qp->stats.dequeued_count += resp_counter;
 669                 rx_queue->nb_processed_responses += resp_counter;
 670                 tmp_qp->inflights16 -= resp_counter;
 671
 672                 if (rx_queue->nb_processed_responses >
 673                                                 QAT_CSR_HEAD_WRITE_THRESH)
 674                         rxq_free_desc(tmp_qp, rx_queue);
 675         }
 676         /* also check if tail needs to be advanced */
 677         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
 678                 tx_queue->tail != tx_queue->csr_tail) {
 679                 txq_write_tail(tmp_qp, tx_queue);
 680         }
 681         return resp_counter;
 682 }
 683
 684 __rte_weak int
 685 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
 686 {
 687         return  0;
 688 }