drivers/net/ena/base/ena_com.c

   1 /*-
   2 * BSD LICENSE
   3 *
   4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
   5 * All rights reserved.
   6 *
   7 * Redistribution and use in source and binary forms, with or without
   8 * modification, are permitted provided that the following conditions
   9 * are met:
  10 *
  11 * * Redistributions of source code must retain the above copyright
  12 * notice, this list of conditions and the following disclaimer.
  13 * * Redistributions in binary form must reproduce the above copyright
  14 * notice, this list of conditions and the following disclaimer in
  15 * the documentation and/or other materials provided with the
  16 * distribution.
  17 * * Neither the name of copyright holder nor the names of its
  18 * contributors may be used to endorse or promote products derived
  19 * from this software without specific prior written permission.
  20 *
  21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32 */
  33
  34 #include "ena_com.h"
  35
  36 /*****************************************************************************/
  37 /*****************************************************************************/
  38
  39 /* Timeout in micro-sec */
  40 #define ADMIN_CMD_TIMEOUT_US (1000000)
  41
  42 #define ENA_ASYNC_QUEUE_DEPTH 4
  43 #define ENA_ADMIN_QUEUE_DEPTH 32
  44
  45 #define ENA_EXTENDED_STAT_GET_FUNCT(_funct_queue) (_funct_queue & 0xFFFF)
  46 #define ENA_EXTENDED_STAT_GET_QUEUE(_funct_queue) (_funct_queue >> 16)
  47
  48 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
  49                 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
  50                 | (ENA_COMMON_SPEC_VERSION_MINOR))
  51
  52 #define ENA_CTRL_MAJOR          0
  53 #define ENA_CTRL_MINOR          0
  54 #define ENA_CTRL_SUB_MINOR      1
  55
  56 #define MIN_ENA_CTRL_VER \
  57         (((ENA_CTRL_MAJOR) << \
  58         (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
  59         ((ENA_CTRL_MINOR) << \
  60         (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
  61         (ENA_CTRL_SUB_MINOR))
  62
  63 #define ENA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
  64 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
  65
  66 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
  67
  68 static int ena_alloc_cnt;
  69
  70 /*****************************************************************************/
  71 /*****************************************************************************/
  72 /*****************************************************************************/
  73
  74 enum ena_cmd_status {
  75         ENA_CMD_SUBMITTED,
  76         ENA_CMD_COMPLETED,
  77         /* Abort - canceled by the driver */
  78         ENA_CMD_ABORTED,
  79 };
  80
  81 struct ena_comp_ctx {
  82         ena_wait_event_t wait_event;
  83         struct ena_admin_acq_entry *user_cqe;
  84         u32 comp_size;
  85         enum ena_cmd_status status;
  86         /* status from the device */
  87         u8 comp_status;
  88         u8 cmd_opcode;
  89         bool occupied;
  90 };
  91
  92 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
  93                                        struct ena_common_mem_addr *ena_addr,
  94                                        dma_addr_t addr)
  95 {
  96         if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
  97                 ena_trc_err("dma address has more bits that the device supports\n");
  98                 return ENA_COM_INVAL;
  99         }
 100
 101         ena_addr->mem_addr_low = (u32)addr;
 102         ena_addr->mem_addr_high =
 103                 ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 32)) >> 32);
 104
 105         return 0;
 106 }
 107
 108 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
 109 {
 110         ENA_MEM_ALLOC_COHERENT(queue->q_dmadev,
 111                                ADMIN_SQ_SIZE(queue->q_depth),
 112                                queue->sq.entries,
 113                                queue->sq.dma_addr,
 114                                queue->sq.mem_handle);
 115
 116         if (!queue->sq.entries) {
 117                 ena_trc_err("memory allocation failed");
 118                 return ENA_COM_NO_MEM;
 119         }
 120
 121         queue->sq.head = 0;
 122         queue->sq.tail = 0;
 123         queue->sq.phase = 1;
 124
 125         queue->sq.db_addr = NULL;
 126
 127         return 0;
 128 }
 129
 130 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
 131 {
 132         ENA_MEM_ALLOC_COHERENT(queue->q_dmadev,
 133                                ADMIN_CQ_SIZE(queue->q_depth),
 134                                queue->cq.entries,
 135                                queue->cq.dma_addr,
 136                                queue->cq.mem_handle);
 137
 138         if (!queue->cq.entries)  {
 139                 ena_trc_err("memory allocation failed");
 140                 return ENA_COM_NO_MEM;
 141         }
 142
 143         queue->cq.head = 0;
 144         queue->cq.phase = 1;
 145
 146         return 0;
 147 }
 148
 149 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
 150                                    struct ena_aenq_handlers *aenq_handlers)
 151 {
 152         u32 addr_low, addr_high, aenq_caps;
 153
 154         dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
 155         ENA_MEM_ALLOC_COHERENT(dev->dmadev,
 156                                ADMIN_AENQ_SIZE(dev->aenq.q_depth),
 157                                dev->aenq.entries,
 158                                dev->aenq.dma_addr,
 159                                dev->aenq.mem_handle);
 160
 161         if (!dev->aenq.entries) {
 162                 ena_trc_err("memory allocation failed");
 163                 return ENA_COM_NO_MEM;
 164         }
 165
 166         dev->aenq.head = dev->aenq.q_depth;
 167         dev->aenq.phase = 1;
 168
 169         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(dev->aenq.dma_addr);
 170         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(dev->aenq.dma_addr);
 171
 172         ENA_REG_WRITE32(addr_low, (unsigned char *)dev->reg_bar
 173                         + ENA_REGS_AENQ_BASE_LO_OFF);
 174         ENA_REG_WRITE32(addr_high, (unsigned char *)dev->reg_bar
 175                         + ENA_REGS_AENQ_BASE_HI_OFF);
 176
 177         aenq_caps = 0;
 178         aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
 179         aenq_caps |= (sizeof(struct ena_admin_aenq_entry) <<
 180                 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
 181                 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
 182
 183         ENA_REG_WRITE32(aenq_caps, (unsigned char *)dev->reg_bar
 184                         + ENA_REGS_AENQ_CAPS_OFF);
 185
 186         if (unlikely(!aenq_handlers))
 187                 ena_trc_err("aenq handlers pointer is NULL\n");
 188
 189         dev->aenq.aenq_handlers = aenq_handlers;
 190
 191         return 0;
 192 }
 193
 194 static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
 195                                      struct ena_comp_ctx *comp_ctx)
 196 {
 197         comp_ctx->occupied = false;
 198         ATOMIC32_DEC(&queue->outstanding_cmds);
 199 }
 200
 201 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
 202                                           u16 command_id, bool capture)
 203 {
 204         ENA_ASSERT(command_id < queue->q_depth,
 205                    "command id is larger than the queue size. cmd_id: %u queue size %d\n",
 206                    command_id, queue->q_depth);
 207
 208         ENA_ASSERT(!(queue->comp_ctx[command_id].occupied && capture),
 209                    "Completion context is occupied");
 210
 211         if (capture) {
 212                 ATOMIC32_INC(&queue->outstanding_cmds);
 213                 queue->comp_ctx[command_id].occupied = true;
 214         }
 215
 216         return &queue->comp_ctx[command_id];
 217 }
 218
 219 static struct ena_comp_ctx *
 220 __ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 221                            struct ena_admin_aq_entry *cmd,
 222                            size_t cmd_size_in_bytes,
 223                            struct ena_admin_acq_entry *comp,
 224                            size_t comp_size_in_bytes)
 225 {
 226         struct ena_comp_ctx *comp_ctx;
 227         u16 tail_masked, cmd_id;
 228         u16 queue_size_mask;
 229         u16 cnt;
 230
 231         queue_size_mask = admin_queue->q_depth - 1;
 232
 233         tail_masked = admin_queue->sq.tail & queue_size_mask;
 234
 235         /* In case of queue FULL */
 236         cnt = admin_queue->sq.tail - admin_queue->sq.head;
 237         if (cnt >= admin_queue->q_depth) {
 238                 ena_trc_dbg("admin queue is FULL (tail %d head %d depth: %d)\n",
 239                             admin_queue->sq.tail,
 240                             admin_queue->sq.head,
 241                             admin_queue->q_depth);
 242                 admin_queue->stats.out_of_space++;
 243                 return ERR_PTR(ENA_COM_NO_SPACE);
 244         }
 245
 246         cmd_id = admin_queue->curr_cmd_id;
 247
 248         cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
 249                 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
 250
 251         cmd->aq_common_descriptor.command_id |= cmd_id &
 252                 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 253
 254         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
 255
 256         comp_ctx->status = ENA_CMD_SUBMITTED;
 257         comp_ctx->comp_size = (u32)comp_size_in_bytes;
 258         comp_ctx->user_cqe = comp;
 259         comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
 260
 261         ENA_WAIT_EVENT_CLEAR(comp_ctx->wait_event);
 262
 263         memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
 264
 265         admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
 266                 queue_size_mask;
 267
 268         admin_queue->sq.tail++;
 269         admin_queue->stats.submitted_cmd++;
 270
 271         if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
 272                 admin_queue->sq.phase = !admin_queue->sq.phase;
 273
 274         ENA_REG_WRITE32(admin_queue->sq.tail, admin_queue->sq.db_addr);
 275
 276         return comp_ctx;
 277 }
 278
 279 static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
 280 {
 281         size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
 282         struct ena_comp_ctx *comp_ctx;
 283         u16 i;
 284
 285         queue->comp_ctx = ENA_MEM_ALLOC(queue->q_dmadev, size);
 286         if (unlikely(!queue->comp_ctx)) {
 287                 ena_trc_err("memory allocation failed");
 288                 return ENA_COM_NO_MEM;
 289         }
 290
 291         for (i = 0; i < queue->q_depth; i++) {
 292                 comp_ctx = get_comp_ctxt(queue, i, false);
 293                 ENA_WAIT_EVENT_INIT(comp_ctx->wait_event);
 294         }
 295
 296         return 0;
 297 }
 298
 299 static struct ena_comp_ctx *
 300 ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
 301                          struct ena_admin_aq_entry *cmd,
 302                          size_t cmd_size_in_bytes,
 303                          struct ena_admin_acq_entry *comp,
 304                          size_t comp_size_in_bytes)
 305 {
 306         unsigned long flags = 0;
 307         struct ena_comp_ctx *comp_ctx;
 308
 309         ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 310         if (unlikely(!admin_queue->running_state)) {
 311                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 312                 return ERR_PTR(ENA_COM_NO_DEVICE);
 313         }
 314         comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
 315                                               cmd_size_in_bytes,
 316                                               comp,
 317                                               comp_size_in_bytes);
 318         ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 319
 320         return comp_ctx;
 321 }
 322
 323 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
 324                               struct ena_com_io_sq *io_sq)
 325 {
 326         size_t size;
 327
 328         memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
 329
 330         io_sq->desc_entry_size =
 331                 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 332                 sizeof(struct ena_eth_io_tx_desc) :
 333                 sizeof(struct ena_eth_io_rx_desc);
 334
 335         size = io_sq->desc_entry_size * io_sq->q_depth;
 336
 337         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 338                 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 339                                        size,
 340                                        io_sq->desc_addr.virt_addr,
 341                                        io_sq->desc_addr.phys_addr,
 342                                        io_sq->desc_addr.mem_handle);
 343         else
 344                 io_sq->desc_addr.virt_addr =
 345                         ENA_MEM_ALLOC(ena_dev->dmadev, size);
 346
 347         if (!io_sq->desc_addr.virt_addr) {
 348                 ena_trc_err("memory allocation failed");
 349                 return ENA_COM_NO_MEM;
 350         }
 351
 352         io_sq->tail = 0;
 353         io_sq->next_to_comp = 0;
 354         io_sq->phase = 1;
 355
 356         return 0;
 357 }
 358
 359 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
 360                               struct ena_com_io_cq *io_cq)
 361 {
 362         size_t size;
 363
 364         memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
 365
 366         /* Use the basic completion descriptor for Rx */
 367         io_cq->cdesc_entry_size_in_bytes =
 368                 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
 369                 sizeof(struct ena_eth_io_tx_cdesc) :
 370                 sizeof(struct ena_eth_io_rx_cdesc_base);
 371
 372         size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 373
 374         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 375                                size,
 376                                io_cq->cdesc_addr.virt_addr,
 377                                io_cq->cdesc_addr.phys_addr,
 378                                io_cq->cdesc_addr.mem_handle);
 379
 380         if (!io_cq->cdesc_addr.virt_addr) {
 381                 ena_trc_err("memory allocation failed");
 382                 return ENA_COM_NO_MEM;
 383         }
 384
 385         io_cq->phase = 1;
 386         io_cq->head = 0;
 387
 388         return 0;
 389 }
 390
 391 static void
 392 ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
 393                                        struct ena_admin_acq_entry *cqe)
 394 {
 395         struct ena_comp_ctx *comp_ctx;
 396         u16 cmd_id;
 397
 398         cmd_id = cqe->acq_common_descriptor.command &
 399                 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
 400
 401         comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
 402
 403         comp_ctx->status = ENA_CMD_COMPLETED;
 404         comp_ctx->comp_status = cqe->acq_common_descriptor.status;
 405
 406         if (comp_ctx->user_cqe)
 407                 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
 408
 409         if (!admin_queue->polling)
 410                 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event);
 411 }
 412
 413 static void
 414 ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
 415 {
 416         struct ena_admin_acq_entry *cqe = NULL;
 417         u16 comp_num = 0;
 418         u16 head_masked;
 419         u8 phase;
 420
 421         head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
 422         phase = admin_queue->cq.phase;
 423
 424         cqe = &admin_queue->cq.entries[head_masked];
 425
 426         /* Go over all the completions */
 427         while ((cqe->acq_common_descriptor.flags &
 428                         ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
 429                 /* Do not read the rest of the completion entry before the
 430                  * phase bit was validated
 431                  */
 432                 rmb();
 433                 ena_com_handle_single_admin_completion(admin_queue, cqe);
 434
 435                 head_masked++;
 436                 comp_num++;
 437                 if (unlikely(head_masked == admin_queue->q_depth)) {
 438                         head_masked = 0;
 439                         phase = !phase;
 440                 }
 441
 442                 cqe = &admin_queue->cq.entries[head_masked];
 443         }
 444
 445         admin_queue->cq.head += comp_num;
 446         admin_queue->cq.phase = phase;
 447         admin_queue->sq.head += comp_num;
 448         admin_queue->stats.completed_cmd += comp_num;
 449 }
 450
 451 static int ena_com_comp_status_to_errno(u8 comp_status)
 452 {
 453         if (unlikely(comp_status != 0))
 454                 ena_trc_err("admin command failed[%u]\n", comp_status);
 455
 456         if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
 457                 return ENA_COM_INVAL;
 458
 459         switch (comp_status) {
 460         case ENA_ADMIN_SUCCESS:
 461                 return 0;
 462         case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
 463                 return ENA_COM_NO_MEM;
 464         case ENA_ADMIN_UNSUPPORTED_OPCODE:
 465                 return ENA_COM_PERMISSION;
 466         case ENA_ADMIN_BAD_OPCODE:
 467         case ENA_ADMIN_MALFORMED_REQUEST:
 468         case ENA_ADMIN_ILLEGAL_PARAMETER:
 469         case ENA_ADMIN_UNKNOWN_ERROR:
 470                 return ENA_COM_INVAL;
 471         }
 472
 473         return 0;
 474 }
 475
 476 static int
 477 ena_com_wait_and_process_admin_cq_polling(
 478                 struct ena_comp_ctx *comp_ctx,
 479                 struct ena_com_admin_queue *admin_queue)
 480 {
 481         unsigned long flags = 0;
 482         u64 start_time;
 483         int ret;
 484
 485         start_time = ENA_GET_SYSTEM_USECS();
 486
 487         while (comp_ctx->status == ENA_CMD_SUBMITTED) {
 488                 if ((ENA_GET_SYSTEM_USECS() - start_time) >
 489                     ADMIN_CMD_TIMEOUT_US) {
 490                         ena_trc_err("Wait for completion (polling) timeout\n");
 491                         /* ENA didn't have any completion */
 492                         ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 493                         admin_queue->stats.no_completion++;
 494                         admin_queue->running_state = false;
 495                         ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 496
 497                         ret = ENA_COM_TIMER_EXPIRED;
 498                         goto err;
 499                 }
 500
 501                 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 502                 ena_com_handle_admin_completion(admin_queue);
 503                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 504         }
 505
 506         if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
 507                 ena_trc_err("Command was aborted\n");
 508                 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 509                 admin_queue->stats.aborted_cmd++;
 510                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 511                 ret = ENA_COM_NO_DEVICE;
 512                 goto err;
 513         }
 514
 515         ENA_ASSERT(comp_ctx->status == ENA_CMD_COMPLETED,
 516                    "Invalid comp status %d\n", comp_ctx->status);
 517
 518         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 519 err:
 520         comp_ctxt_release(admin_queue, comp_ctx);
 521         return ret;
 522 }
 523
 524 static int
 525 ena_com_wait_and_process_admin_cq_interrupts(
 526                 struct ena_comp_ctx *comp_ctx,
 527                 struct ena_com_admin_queue *admin_queue)
 528 {
 529         unsigned long flags = 0;
 530         int ret = 0;
 531
 532         ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event,
 533                             ADMIN_CMD_TIMEOUT_US);
 534
 535         /* In case the command wasn't completed find out the root cause.
 536          * There might be 2 kinds of errors
 537          * 1) No completion (timeout reached)
 538          * 2) There is completion but the device didn't get any msi-x interrupt.
 539          */
 540         if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
 541                 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
 542                 ena_com_handle_admin_completion(admin_queue);
 543                 admin_queue->stats.no_completion++;
 544                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
 545
 546                 if (comp_ctx->status == ENA_CMD_COMPLETED)
 547                         ena_trc_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
 548                                     comp_ctx->cmd_opcode);
 549                 else
 550                         ena_trc_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
 551                                     comp_ctx->cmd_opcode, comp_ctx->status);
 552
 553                 admin_queue->running_state = false;
 554                 ret = ENA_COM_TIMER_EXPIRED;
 555                 goto err;
 556         }
 557
 558         ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
 559 err:
 560         comp_ctxt_release(admin_queue, comp_ctx);
 561         return ret;
 562 }
 563
 564 /* This method read the hardware device register through posting writes
 565  * and waiting for response
 566  * On timeout the function will return ENA_MMIO_READ_TIMEOUT
 567  */
 568 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
 569 {
 570         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
 571         volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
 572                 mmio_read->read_resp;
 573         u32 mmio_read_reg, ret;
 574         unsigned long flags = 0;
 575         int i;
 576
 577         ENA_MIGHT_SLEEP();
 578
 579         /* If readless is disabled, perform regular read */
 580         if (!mmio_read->readless_supported)
 581                 return ENA_REG_READ32((unsigned char *)ena_dev->reg_bar +
 582                                       offset);
 583
 584         ENA_SPINLOCK_LOCK(mmio_read->lock, flags);
 585         mmio_read->seq_num++;
 586
 587         read_resp->req_id = mmio_read->seq_num + 0xDEAD;
 588         mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
 589                         ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
 590         mmio_read_reg |= mmio_read->seq_num &
 591                         ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
 592
 593         /* make sure read_resp->req_id get updated before the hw can write
 594          * there
 595          */
 596         wmb();
 597
 598         ENA_REG_WRITE32(mmio_read_reg, (unsigned char *)ena_dev->reg_bar
 599                         + ENA_REGS_MMIO_REG_READ_OFF);
 600
 601         for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) {
 602                 if (read_resp->req_id == mmio_read->seq_num)
 603                         break;
 604
 605                 ENA_UDELAY(1);
 606         }
 607
 608         if (unlikely(i == ENA_REG_READ_TIMEOUT)) {
 609                 ena_trc_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
 610                             mmio_read->seq_num,
 611                             offset,
 612                             read_resp->req_id,
 613                             read_resp->reg_off);
 614                 ret = ENA_MMIO_READ_TIMEOUT;
 615                 goto err;
 616         }
 617
 618         ENA_ASSERT(read_resp->reg_off == offset,
 619                    "Invalid MMIO read return value");
 620
 621         ret = read_resp->reg_val;
 622 err:
 623         ENA_SPINLOCK_UNLOCK(mmio_read->lock, flags);
 624
 625         return ret;
 626 }
 627
 628 /* There are two types to wait for completion.
 629  * Polling mode - wait until the completion is available.
 630  * Async mode - wait on wait queue until the completion is ready
 631  * (or the timeout expired).
 632  * It is expected that the IRQ called ena_com_handle_admin_completion
 633  * to mark the completions.
 634  */
 635 static int
 636 ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
 637                                   struct ena_com_admin_queue *admin_queue)
 638 {
 639         if (admin_queue->polling)
 640                 return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
 641                                                                  admin_queue);
 642
 643         return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
 644                                                             admin_queue);
 645 }
 646
 647 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
 648                                  struct ena_com_io_sq *io_sq)
 649 {
 650         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 651         struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
 652         struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
 653         u8 direction;
 654         int ret;
 655
 656         memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
 657
 658         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 659                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 660         else
 661                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 662
 663         destroy_cmd.sq.sq_identity |= (direction <<
 664                 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
 665                 ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
 666
 667         destroy_cmd.sq.sq_idx = io_sq->idx;
 668         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
 669
 670         ret = ena_com_execute_admin_command(
 671                         admin_queue,
 672                         (struct ena_admin_aq_entry *)&destroy_cmd,
 673                         sizeof(destroy_cmd),
 674                         (struct ena_admin_acq_entry *)&destroy_resp,
 675                         sizeof(destroy_resp));
 676
 677         if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
 678                 ena_trc_err("failed to destroy io sq error: %d\n", ret);
 679
 680         return ret;
 681 }
 682
 683 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
 684                                   struct ena_com_io_sq *io_sq,
 685                                   struct ena_com_io_cq *io_cq)
 686 {
 687         size_t size;
 688
 689         if (io_cq->cdesc_addr.virt_addr) {
 690                 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
 691
 692                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 693                                       size,
 694                                       io_cq->cdesc_addr.virt_addr,
 695                                       io_cq->cdesc_addr.phys_addr,
 696                                       io_cq->cdesc_addr.mem_handle);
 697
 698                 io_cq->cdesc_addr.virt_addr = NULL;
 699         }
 700
 701         if (io_sq->desc_addr.virt_addr) {
 702                 size = io_sq->desc_entry_size * io_sq->q_depth;
 703
 704                 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 705                         ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 706                                               size,
 707                                               io_sq->desc_addr.virt_addr,
 708                                               io_sq->desc_addr.phys_addr,
 709                                               io_sq->desc_addr.mem_handle);
 710                 else
 711                         ENA_MEM_FREE(ena_dev->dmadev,
 712                                      io_sq->desc_addr.virt_addr);
 713
 714                 io_sq->desc_addr.virt_addr = NULL;
 715         }
 716 }
 717
 718 static int wait_for_reset_state(struct ena_com_dev *ena_dev,
 719                                 u32 timeout, u16 exp_state)
 720 {
 721         u32 val, i;
 722
 723         for (i = 0; i < timeout; i++) {
 724                 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
 725
 726                 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
 727                         ena_trc_err("Reg read timeout occurred\n");
 728                         return ENA_COM_TIMER_EXPIRED;
 729                 }
 730
 731                 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
 732                         exp_state)
 733                         return 0;
 734
 735                 /* The resolution of the timeout is 100ms */
 736                 ENA_MSLEEP(100);
 737         }
 738
 739         return ENA_COM_TIMER_EXPIRED;
 740 }
 741
 742 static bool
 743 ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
 744                                    enum ena_admin_aq_feature_id feature_id)
 745 {
 746         u32 feature_mask = 1 << feature_id;
 747
 748         /* Device attributes is always supported */
 749         if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
 750             !(ena_dev->supported_features & feature_mask))
 751                 return false;
 752
 753         return true;
 754 }
 755
 756 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
 757                                   struct ena_admin_get_feat_resp *get_resp,
 758                                   enum ena_admin_aq_feature_id feature_id,
 759                                   dma_addr_t control_buf_dma_addr,
 760                                   u32 control_buff_size)
 761 {
 762         struct ena_com_admin_queue *admin_queue;
 763         struct ena_admin_get_feat_cmd get_cmd;
 764         int ret;
 765
 766         if (!ena_dev) {
 767                 ena_trc_err("%s : ena_dev is NULL\n", __func__);
 768                 return ENA_COM_NO_DEVICE;
 769         }
 770
 771         if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
 772                 ena_trc_info("Feature %d isn't supported\n", feature_id);
 773                 return ENA_COM_PERMISSION;
 774         }
 775
 776         memset(&get_cmd, 0x0, sizeof(get_cmd));
 777         admin_queue = &ena_dev->admin_queue;
 778
 779         get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
 780
 781         if (control_buff_size)
 782                 get_cmd.aq_common_descriptor.flags =
 783                         ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
 784         else
 785                 get_cmd.aq_common_descriptor.flags = 0;
 786
 787         ret = ena_com_mem_addr_set(ena_dev,
 788                                    &get_cmd.control_buffer.address,
 789                                    control_buf_dma_addr);
 790         if (unlikely(ret)) {
 791                 ena_trc_err("memory address set failed\n");
 792                 return ret;
 793         }
 794
 795         get_cmd.control_buffer.length = control_buff_size;
 796
 797         get_cmd.feat_common.feature_id = feature_id;
 798
 799         ret = ena_com_execute_admin_command(admin_queue,
 800                                             (struct ena_admin_aq_entry *)
 801                                             &get_cmd,
 802                                             sizeof(get_cmd),
 803                                             (struct ena_admin_acq_entry *)
 804                                             get_resp,
 805                                             sizeof(*get_resp));
 806
 807         if (unlikely(ret))
 808                 ena_trc_err("Failed to submit get_feature command %d error: %d\n",
 809                             feature_id, ret);
 810
 811         return ret;
 812 }
 813
 814 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
 815                                struct ena_admin_get_feat_resp *get_resp,
 816                                enum ena_admin_aq_feature_id feature_id)
 817 {
 818         return ena_com_get_feature_ex(ena_dev,
 819                                       get_resp,
 820                                       feature_id,
 821                                       0,
 822                                       0);
 823 }
 824
 825 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
 826 {
 827         struct ena_rss *rss = &ena_dev->rss;
 828
 829         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 830                                sizeof(*rss->hash_key),
 831                                rss->hash_key,
 832                                rss->hash_key_dma_addr,
 833                                rss->hash_key_mem_handle);
 834
 835         if (unlikely(!rss->hash_key))
 836                 return ENA_COM_NO_MEM;
 837
 838         return 0;
 839 }
 840
 841 static int ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
 842 {
 843         struct ena_rss *rss = &ena_dev->rss;
 844
 845         if (rss->hash_key)
 846                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 847                                       sizeof(*rss->hash_key),
 848                                       rss->hash_key,
 849                                       rss->hash_key_dma_addr,
 850                                       rss->hash_key_mem_handle);
 851         rss->hash_key = NULL;
 852         return 0;
 853 }
 854
 855 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
 856 {
 857         struct ena_rss *rss = &ena_dev->rss;
 858
 859         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 860                                sizeof(*rss->hash_ctrl),
 861                                rss->hash_ctrl,
 862                                rss->hash_ctrl_dma_addr,
 863                                rss->hash_ctrl_mem_handle);
 864
 865         return 0;
 866 }
 867
 868 static int ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
 869 {
 870         struct ena_rss *rss = &ena_dev->rss;
 871
 872         if (rss->hash_ctrl)
 873                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 874                                       sizeof(*rss->hash_ctrl),
 875                                       rss->hash_ctrl,
 876                                       rss->hash_ctrl_dma_addr,
 877                                       rss->hash_ctrl_mem_handle);
 878         rss->hash_ctrl = NULL;
 879
 880         return 0;
 881 }
 882
 883 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
 884                                            u16 log_size)
 885 {
 886         struct ena_rss *rss = &ena_dev->rss;
 887         struct ena_admin_get_feat_resp get_resp;
 888         size_t tbl_size;
 889         int ret;
 890
 891         ret = ena_com_get_feature(ena_dev, &get_resp,
 892                                   ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
 893         if (unlikely(ret))
 894                 return ret;
 895
 896         if ((get_resp.u.ind_table.min_size > log_size) ||
 897             (get_resp.u.ind_table.max_size < log_size)) {
 898                 ena_trc_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
 899                             1 << log_size,
 900                             1 << get_resp.u.ind_table.min_size,
 901                             1 << get_resp.u.ind_table.max_size);
 902                 return ENA_COM_INVAL;
 903         }
 904
 905         tbl_size = (1 << log_size) *
 906                 sizeof(struct ena_admin_rss_ind_table_entry);
 907
 908         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
 909                                tbl_size,
 910                                rss->rss_ind_tbl,
 911                                rss->rss_ind_tbl_dma_addr,
 912                                rss->rss_ind_tbl_mem_handle);
 913         if (unlikely(!rss->rss_ind_tbl))
 914                 goto mem_err1;
 915
 916         tbl_size = (1 << log_size) * sizeof(u16);
 917         rss->host_rss_ind_tbl =
 918                 ENA_MEM_ALLOC(ena_dev->dmadev, tbl_size);
 919         if (unlikely(!rss->host_rss_ind_tbl))
 920                 goto mem_err2;
 921
 922         rss->tbl_log_size = log_size;
 923
 924         return 0;
 925
 926 mem_err2:
 927         tbl_size = (1 << log_size) *
 928                 sizeof(struct ena_admin_rss_ind_table_entry);
 929
 930         ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 931                               tbl_size,
 932                               rss->rss_ind_tbl,
 933                               rss->rss_ind_tbl_dma_addr,
 934                               rss->rss_ind_tbl_mem_handle);
 935         rss->rss_ind_tbl = NULL;
 936 mem_err1:
 937         rss->tbl_log_size = 0;
 938         return ENA_COM_NO_MEM;
 939 }
 940
 941 static int ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
 942 {
 943         struct ena_rss *rss = &ena_dev->rss;
 944         size_t tbl_size = (1 << rss->tbl_log_size) *
 945                 sizeof(struct ena_admin_rss_ind_table_entry);
 946
 947         if (rss->rss_ind_tbl)
 948                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
 949                                       tbl_size,
 950                                       rss->rss_ind_tbl,
 951                                       rss->rss_ind_tbl_dma_addr,
 952                                       rss->rss_ind_tbl_mem_handle);
 953         rss->rss_ind_tbl = NULL;
 954
 955         if (rss->host_rss_ind_tbl)
 956                 ENA_MEM_FREE(ena_dev->dmadev, rss->host_rss_ind_tbl);
 957         rss->host_rss_ind_tbl = NULL;
 958
 959         return 0;
 960 }
 961
 962 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
 963                                 struct ena_com_io_sq *io_sq, u16 cq_idx)
 964 {
 965         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
 966         struct ena_admin_aq_create_sq_cmd create_cmd;
 967         struct ena_admin_acq_create_sq_resp_desc cmd_completion;
 968         u8 direction;
 969         int ret;
 970
 971         memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd));
 972
 973         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
 974
 975         if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
 976                 direction = ENA_ADMIN_SQ_DIRECTION_TX;
 977         else
 978                 direction = ENA_ADMIN_SQ_DIRECTION_RX;
 979
 980         create_cmd.sq_identity |= (direction <<
 981                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
 982                 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
 983
 984         create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
 985                 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
 986
 987         create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
 988                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
 989                 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
 990
 991         create_cmd.sq_caps_3 |=
 992                 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
 993
 994         create_cmd.cq_idx = cq_idx;
 995         create_cmd.sq_depth = io_sq->q_depth;
 996
 997         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
 998                 ret = ena_com_mem_addr_set(ena_dev,
 999                                            &create_cmd.sq_ba,
1000                                            io_sq->desc_addr.phys_addr);
1001                 if (unlikely(ret)) {
1002                         ena_trc_err("memory address set failed\n");
1003                         return ret;
1004                 }
1005         }
1006
1007         ret = ena_com_execute_admin_command(
1008                         admin_queue,
1009                         (struct ena_admin_aq_entry *)&create_cmd,
1010                         sizeof(create_cmd),
1011                         (struct ena_admin_acq_entry *)&cmd_completion,
1012                         sizeof(cmd_completion));
1013         if (unlikely(ret)) {
1014                 ena_trc_err("Failed to create IO SQ. error: %d\n", ret);
1015                 return ret;
1016         }
1017
1018         io_sq->idx = cmd_completion.sq_idx;
1019
1020         io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1021                 (uintptr_t)cmd_completion.sq_doorbell_offset);
1022
1023         if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1024                 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1025                                 + cmd_completion.llq_headers_offset);
1026
1027                 io_sq->desc_addr.pbuf_dev_addr =
1028                         (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1029                         cmd_completion.llq_descriptors_offset);
1030         }
1031
1032         ena_trc_dbg("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
1033
1034         return ret;
1035 }
1036
1037 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1038 {
1039         struct ena_rss *rss = &ena_dev->rss;
1040         struct ena_com_io_sq *io_sq;
1041         u16 qid;
1042         int i;
1043
1044         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1045                 qid = rss->host_rss_ind_tbl[i];
1046                 if (qid >= ENA_TOTAL_NUM_QUEUES)
1047                         return ENA_COM_INVAL;
1048
1049                 io_sq = &ena_dev->io_sq_queues[qid];
1050
1051                 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1052                         return ENA_COM_INVAL;
1053
1054                 rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1055         }
1056
1057         return 0;
1058 }
1059
1060 static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
1061 {
1062         u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { -1 };
1063         struct ena_rss *rss = &ena_dev->rss;
1064         u16 idx, i;
1065
1066         for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
1067                 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
1068
1069         for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1070                 idx = rss->rss_ind_tbl[i].cq_idx;
1071                 if (idx > ENA_TOTAL_NUM_QUEUES)
1072                         return ENA_COM_INVAL;
1073
1074                 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
1075                         return ENA_COM_INVAL;
1076
1077                 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
1078         }
1079
1080         return 0;
1081 }
1082
1083 static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
1084 {
1085         size_t size;
1086
1087         size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
1088
1089         ena_dev->intr_moder_tbl = ENA_MEM_ALLOC(ena_dev->dmadev, size);
1090         if (!ena_dev->intr_moder_tbl)
1091                 return ENA_COM_NO_MEM;
1092
1093         ena_com_config_default_interrupt_moderation_table(ena_dev);
1094
1095         return 0;
1096 }
1097
1098 static void
1099 ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1100                                      unsigned int intr_delay_resolution)
1101 {
1102         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
1103         unsigned int i;
1104
1105         if (!intr_delay_resolution) {
1106                 ena_trc_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1107                 intr_delay_resolution = 1;
1108         }
1109         ena_dev->intr_delay_resolution = intr_delay_resolution;
1110
1111         /* update Rx */
1112         for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
1113                 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
1114
1115         /* update Tx */
1116         ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
1117 }
1118
1119 /*****************************************************************************/
1120 /*******************************      API       ******************************/
1121 /*****************************************************************************/
1122
1123 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1124                                   struct ena_admin_aq_entry *cmd,
1125                                   size_t cmd_size,
1126                                   struct ena_admin_acq_entry *comp,
1127                                   size_t comp_size)
1128 {
1129         struct ena_comp_ctx *comp_ctx;
1130         int ret = 0;
1131
1132         comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1133                                             comp, comp_size);
1134         if (unlikely(IS_ERR(comp_ctx))) {
1135                 ena_trc_err("Failed to submit command [%ld]\n",
1136                             PTR_ERR(comp_ctx));
1137                 return PTR_ERR(comp_ctx);
1138         }
1139
1140         ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1141         if (unlikely(ret)) {
1142                 if (admin_queue->running_state)
1143                         ena_trc_err("Failed to process command. ret = %d\n",
1144                                     ret);
1145                 else
1146                         ena_trc_dbg("Failed to process command. ret = %d\n",
1147                                     ret);
1148         }
1149         return ret;
1150 }
1151
1152 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1153                          struct ena_com_io_cq *io_cq)
1154 {
1155         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1156         struct ena_admin_aq_create_cq_cmd create_cmd;
1157         struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1158         int ret;
1159
1160         memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd));
1161
1162         create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1163
1164         create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1165                 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1166         create_cmd.cq_caps_1 |=
1167                 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1168
1169         create_cmd.msix_vector = io_cq->msix_vector;
1170         create_cmd.cq_depth = io_cq->q_depth;
1171
1172         ret = ena_com_mem_addr_set(ena_dev,
1173                                    &create_cmd.cq_ba,
1174                                    io_cq->cdesc_addr.phys_addr);
1175         if (unlikely(ret)) {
1176                 ena_trc_err("memory address set failed\n");
1177                 return ret;
1178         }
1179
1180         ret = ena_com_execute_admin_command(
1181                         admin_queue,
1182                         (struct ena_admin_aq_entry *)&create_cmd,
1183                         sizeof(create_cmd),
1184                         (struct ena_admin_acq_entry *)&cmd_completion,
1185                         sizeof(cmd_completion));
1186         if (unlikely(ret)) {
1187                 ena_trc_err("Failed to create IO CQ. error: %d\n", ret);
1188                 return ret;
1189         }
1190
1191         io_cq->idx = cmd_completion.cq_idx;
1192         io_cq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1193                 cmd_completion.cq_doorbell_offset);
1194
1195         if (io_cq->q_depth != cmd_completion.cq_actual_depth) {
1196                 ena_trc_err("completion actual queue size (%d) is differ from requested size (%d)\n",
1197                             cmd_completion.cq_actual_depth, io_cq->q_depth);
1198                 ena_com_destroy_io_cq(ena_dev, io_cq);
1199                 return ENA_COM_NO_SPACE;
1200         }
1201
1202         io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1203                 cmd_completion.cq_interrupt_unmask_register);
1204
1205         if (cmd_completion.cq_head_db_offset)
1206                 io_cq->cq_head_db_reg =
1207                         (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1208                         cmd_completion.cq_head_db_offset);
1209
1210         ena_trc_dbg("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
1211
1212         return ret;
1213 }
1214
1215 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1216                             struct ena_com_io_sq **io_sq,
1217                             struct ena_com_io_cq **io_cq)
1218 {
1219         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1220                 ena_trc_err("Invalid queue number %d but the max is %d\n",
1221                             qid, ENA_TOTAL_NUM_QUEUES);
1222                 return ENA_COM_INVAL;
1223         }
1224
1225         *io_sq = &ena_dev->io_sq_queues[qid];
1226         *io_cq = &ena_dev->io_cq_queues[qid];
1227
1228         return 0;
1229 }
1230
1231 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1232 {
1233         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1234         struct ena_comp_ctx *comp_ctx;
1235         u16 i;
1236
1237         if (!admin_queue->comp_ctx)
1238                 return;
1239
1240         for (i = 0; i < admin_queue->q_depth; i++) {
1241                 comp_ctx = get_comp_ctxt(admin_queue, i, false);
1242                 comp_ctx->status = ENA_CMD_ABORTED;
1243
1244                 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event);
1245         }
1246 }
1247
1248 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1249 {
1250         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1251         unsigned long flags = 0;
1252
1253         ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
1254         while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {
1255                 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
1256                 ENA_MSLEEP(20);
1257                 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
1258         }
1259         ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
1260 }
1261
1262 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1263                           struct ena_com_io_cq *io_cq)
1264 {
1265         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1266         struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1267         struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1268         int ret;
1269
1270         memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
1271
1272         destroy_cmd.cq_idx = io_cq->idx;
1273         destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1274
1275         ret = ena_com_execute_admin_command(
1276                         admin_queue,
1277                         (struct ena_admin_aq_entry *)&destroy_cmd,
1278                         sizeof(destroy_cmd),
1279                         (struct ena_admin_acq_entry *)&destroy_resp,
1280                         sizeof(destroy_resp));
1281
1282         if (unlikely(ret && (ret != ENA_COM_NO_DEVICE)))
1283                 ena_trc_err("Failed to destroy IO CQ. error: %d\n", ret);
1284
1285         return ret;
1286 }
1287
1288 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1289 {
1290         return ena_dev->admin_queue.running_state;
1291 }
1292
1293 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1294 {
1295         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1296         unsigned long flags = 0;
1297
1298         ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);
1299         ena_dev->admin_queue.running_state = state;
1300         ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
1301 }
1302
1303 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1304 {
1305         u16 depth = ena_dev->aenq.q_depth;
1306
1307         ENA_ASSERT(ena_dev->aenq.head == depth, "Invliad AENQ state\n");
1308
1309         /* Init head_db to mark that all entries in the queue
1310          * are initially available
1311          */
1312         ENA_REG_WRITE32(depth, (unsigned char *)ena_dev->reg_bar
1313                         + ENA_REGS_AENQ_HEAD_DB_OFF);
1314 }
1315
1316 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1317 {
1318         struct ena_com_admin_queue *admin_queue;
1319         struct ena_admin_set_feat_cmd cmd;
1320         struct ena_admin_set_feat_resp resp;
1321         struct ena_admin_get_feat_resp get_resp;
1322         int ret = 0;
1323
1324         if (unlikely(!ena_dev)) {
1325                 ena_trc_err("%s : ena_dev is NULL\n", __func__);
1326                 return ENA_COM_NO_DEVICE;
1327         }
1328
1329         ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
1330         if (ret) {
1331                 ena_trc_info("Can't get aenq configuration\n");
1332                 return ret;
1333         }
1334
1335         if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1336                 ena_trc_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
1337                              get_resp.u.aenq.supported_groups,
1338                              groups_flag);
1339                 return ENA_COM_PERMISSION;
1340         }
1341
1342         memset(&cmd, 0x0, sizeof(cmd));
1343         admin_queue = &ena_dev->admin_queue;
1344
1345         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1346         cmd.aq_common_descriptor.flags = 0;
1347         cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1348         cmd.u.aenq.enabled_groups = groups_flag;
1349
1350         ret = ena_com_execute_admin_command(admin_queue,
1351                                             (struct ena_admin_aq_entry *)&cmd,
1352                                             sizeof(cmd),
1353                                             (struct ena_admin_acq_entry *)&resp,
1354                                             sizeof(resp));
1355
1356         if (unlikely(ret))
1357                 ena_trc_err("Failed to config AENQ ret: %d\n", ret);
1358
1359         return ret;
1360 }
1361
1362 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1363 {
1364         u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1365         int width;
1366
1367         if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1368                 ena_trc_err("Reg read timeout occurred\n");
1369                 return ENA_COM_TIMER_EXPIRED;
1370         }
1371
1372         width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1373                 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1374
1375         ena_trc_dbg("ENA dma width: %d\n", width);
1376
1377         if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1378                 ena_trc_err("DMA width illegal value: %d\n", width);
1379                 return ENA_COM_INVAL;
1380         }
1381
1382         ena_dev->dma_addr_bits = width;
1383
1384         return width;
1385 }
1386
1387 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1388 {
1389         u32 ver;
1390         u32 ctrl_ver;
1391         u32 ctrl_ver_masked;
1392
1393         /* Make sure the ENA version and the controller version are at least
1394          * as the driver expects
1395          */
1396         ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1397         ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1398                                           ENA_REGS_CONTROLLER_VERSION_OFF);
1399
1400         if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1401                      (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1402                 ena_trc_err("Reg read timeout occurred\n");
1403                 return ENA_COM_TIMER_EXPIRED;
1404         }
1405
1406         ena_trc_info("ena device version: %d.%d\n",
1407                      (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1408                      ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1409                      ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1410
1411         if (ver < MIN_ENA_VER) {
1412                 ena_trc_err("ENA version is lower than the minimal version the driver supports\n");
1413                 return -1;
1414         }
1415
1416         ena_trc_info("ena controller version: %d.%d.%d implementation version %d\n",
1417                      (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK)
1418                      >> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1419                      (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK)
1420                      >> ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1421                      (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1422                      (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1423                      ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1424
1425         ctrl_ver_masked =
1426                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1427                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1428                 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1429
1430         /* Validate the ctrl version without the implementation ID */
1431         if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1432                 ena_trc_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1433                 return -1;
1434         }
1435
1436         return 0;
1437 }
1438
1439 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1440 {
1441         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1442
1443         if (!admin_queue)
1444                 return;
1445
1446         if (admin_queue->comp_ctx)
1447                 ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx);
1448         admin_queue->comp_ctx = NULL;
1449
1450         if (admin_queue->sq.entries)
1451                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
1452                                       ADMIN_SQ_SIZE(admin_queue->q_depth),
1453                                       admin_queue->sq.entries,
1454                                       admin_queue->sq.dma_addr,
1455                                       admin_queue->sq.mem_handle);
1456         admin_queue->sq.entries = NULL;
1457
1458         if (admin_queue->cq.entries)
1459                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
1460                                       ADMIN_CQ_SIZE(admin_queue->q_depth),
1461                                       admin_queue->cq.entries,
1462                                       admin_queue->cq.dma_addr,
1463                                       admin_queue->cq.mem_handle);
1464         admin_queue->cq.entries = NULL;
1465
1466         if (ena_dev->aenq.entries)
1467                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
1468                                       ADMIN_AENQ_SIZE(ena_dev->aenq.q_depth),
1469                                       ena_dev->aenq.entries,
1470                                       ena_dev->aenq.dma_addr,
1471                                       ena_dev->aenq.mem_handle);
1472         ena_dev->aenq.entries = NULL;
1473 }
1474
1475 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1476 {
1477         ena_dev->admin_queue.polling = polling;
1478 }
1479
1480 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1481 {
1482         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1483
1484         ENA_SPINLOCK_INIT(mmio_read->lock);
1485         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
1486                                sizeof(*mmio_read->read_resp),
1487                                mmio_read->read_resp,
1488                                mmio_read->read_resp_dma_addr,
1489                                mmio_read->read_resp_mem_handle);
1490         if (unlikely(!mmio_read->read_resp))
1491                 return ENA_COM_NO_MEM;
1492
1493         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1494
1495         mmio_read->read_resp->req_id = 0x0;
1496         mmio_read->seq_num = 0x0;
1497         mmio_read->readless_supported = true;
1498
1499         return 0;
1500 }
1501
1502 void
1503 ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1504 {
1505         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1506
1507         mmio_read->readless_supported = readless_supported;
1508 }
1509
1510 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1511 {
1512         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1513
1514         ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar
1515                         + ENA_REGS_MMIO_RESP_LO_OFF);
1516         ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar
1517                         + ENA_REGS_MMIO_RESP_HI_OFF);
1518
1519         ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
1520                               sizeof(*mmio_read->read_resp),
1521                               mmio_read->read_resp,
1522                               mmio_read->read_resp_dma_addr,
1523                               mmio_read->read_resp_mem_handle);
1524
1525         mmio_read->read_resp = NULL;
1526 }
1527
1528 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1529 {
1530         struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1531         u32 addr_low, addr_high;
1532
1533         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1534         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1535
1536         ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
1537                         + ENA_REGS_MMIO_RESP_LO_OFF);
1538         ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
1539                         + ENA_REGS_MMIO_RESP_HI_OFF);
1540 }
1541
1542 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1543                        struct ena_aenq_handlers *aenq_handlers,
1544                        bool init_spinlock)
1545 {
1546         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1547         u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1548         int ret;
1549
1550         dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1551
1552         if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1553                 ena_trc_err("Reg read timeout occurred\n");
1554                 return ENA_COM_TIMER_EXPIRED;
1555         }
1556
1557         if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1558                 ena_trc_err("Device isn't ready, abort com init\n");
1559                 return -1;
1560         }
1561
1562         admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1563
1564         admin_queue->q_dmadev = ena_dev->dmadev;
1565         admin_queue->polling = false;
1566         admin_queue->curr_cmd_id = 0;
1567
1568         ATOMIC32_SET(&admin_queue->outstanding_cmds, 0);
1569
1570         if (init_spinlock)
1571                 ENA_SPINLOCK_INIT(admin_queue->q_lock);
1572
1573         ret = ena_com_init_comp_ctxt(admin_queue);
1574         if (ret)
1575                 goto error;
1576
1577         ret = ena_com_admin_init_sq(admin_queue);
1578         if (ret)
1579                 goto error;
1580
1581         ret = ena_com_admin_init_cq(admin_queue);
1582         if (ret)
1583                 goto error;
1584
1585         admin_queue->sq.db_addr = (u32 __iomem *)
1586                 ((unsigned char *)ena_dev->reg_bar + ENA_REGS_AQ_DB_OFF);
1587
1588         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1589         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1590
1591         ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
1592                         + ENA_REGS_AQ_BASE_LO_OFF);
1593         ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
1594                         + ENA_REGS_AQ_BASE_HI_OFF);
1595
1596         addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1597         addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1598
1599         ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar
1600                         + ENA_REGS_ACQ_BASE_LO_OFF);
1601         ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar
1602                         + ENA_REGS_ACQ_BASE_HI_OFF);
1603
1604         aq_caps = 0;
1605         aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1606         aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1607                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1608                         ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1609
1610         acq_caps = 0;
1611         acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1612         acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1613                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1614                 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1615
1616         ENA_REG_WRITE32(aq_caps, (unsigned char *)ena_dev->reg_bar
1617                         + ENA_REGS_AQ_CAPS_OFF);
1618         ENA_REG_WRITE32(acq_caps, (unsigned char *)ena_dev->reg_bar
1619                         + ENA_REGS_ACQ_CAPS_OFF);
1620         ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1621         if (ret)
1622                 goto error;
1623
1624         admin_queue->running_state = true;
1625
1626         return 0;
1627 error:
1628         ena_com_admin_destroy(ena_dev);
1629
1630         return ret;
1631 }
1632
1633 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1634                             u16 qid,
1635                             enum queue_direction direction,
1636                             enum ena_admin_placement_policy_type mem_queue_type,
1637                             u32 msix_vector,
1638                             u16 queue_size)
1639 {
1640         struct ena_com_io_sq *io_sq;
1641         struct ena_com_io_cq *io_cq;
1642         int ret = 0;
1643
1644         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1645                 ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n",
1646                             qid, ENA_TOTAL_NUM_QUEUES);
1647                 return ENA_COM_INVAL;
1648         }
1649
1650         io_sq = &ena_dev->io_sq_queues[qid];
1651         io_cq = &ena_dev->io_cq_queues[qid];
1652
1653         memset(io_sq, 0x0, sizeof(struct ena_com_io_sq));
1654         memset(io_cq, 0x0, sizeof(struct ena_com_io_cq));
1655
1656         /* Init CQ */
1657         io_cq->q_depth = queue_size;
1658         io_cq->direction = direction;
1659         io_cq->qid = qid;
1660
1661         io_cq->msix_vector = msix_vector;
1662
1663         io_sq->q_depth = queue_size;
1664         io_sq->direction = direction;
1665         io_sq->qid = qid;
1666
1667         io_sq->mem_queue_type = mem_queue_type;
1668
1669         if (direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1670                 /* header length is limited to 8 bits */
1671                 io_sq->tx_max_header_size =
1672                         ENA_MIN16(ena_dev->tx_max_header_size, SZ_256);
1673
1674         ret = ena_com_init_io_sq(ena_dev, io_sq);
1675         if (ret)
1676                 goto error;
1677         ret = ena_com_init_io_cq(ena_dev, io_cq);
1678         if (ret)
1679                 goto error;
1680
1681         ret = ena_com_create_io_cq(ena_dev, io_cq);
1682         if (ret)
1683                 goto error;
1684
1685         ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1686         if (ret)
1687                 goto destroy_io_cq;
1688
1689         return 0;
1690
1691 destroy_io_cq:
1692         ena_com_destroy_io_cq(ena_dev, io_cq);
1693 error:
1694         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1695         return ret;
1696 }
1697
1698 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1699 {
1700         struct ena_com_io_sq *io_sq;
1701         struct ena_com_io_cq *io_cq;
1702
1703         if (qid >= ENA_TOTAL_NUM_QUEUES) {
1704                 ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n",
1705                             qid, ENA_TOTAL_NUM_QUEUES);
1706                 return;
1707         }
1708
1709         io_sq = &ena_dev->io_sq_queues[qid];
1710         io_cq = &ena_dev->io_cq_queues[qid];
1711
1712         ena_com_destroy_io_sq(ena_dev, io_sq);
1713         ena_com_destroy_io_cq(ena_dev, io_cq);
1714
1715         ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1716 }
1717
1718 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1719                             struct ena_admin_get_feat_resp *resp)
1720 {
1721         return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
1722 }
1723
1724 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1725                               struct ena_com_dev_get_features_ctx *get_feat_ctx)
1726 {
1727         struct ena_admin_get_feat_resp get_resp;
1728         int rc;
1729
1730         rc = ena_com_get_feature(ena_dev, &get_resp,
1731                                  ENA_ADMIN_DEVICE_ATTRIBUTES);
1732         if (rc)
1733                 return rc;
1734
1735         memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1736                sizeof(get_resp.u.dev_attr));
1737         ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1738
1739         rc = ena_com_get_feature(ena_dev, &get_resp,
1740                                  ENA_ADMIN_MAX_QUEUES_NUM);
1741         if (rc)
1742                 return rc;
1743
1744         memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1745                sizeof(get_resp.u.max_queue));
1746         ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
1747
1748         rc = ena_com_get_feature(ena_dev, &get_resp,
1749                                  ENA_ADMIN_AENQ_CONFIG);
1750         if (rc)
1751                 return rc;
1752
1753         memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
1754                sizeof(get_resp.u.aenq));
1755
1756         rc = ena_com_get_feature(ena_dev, &get_resp,
1757                                  ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1758         if (rc)
1759                 return rc;
1760
1761         memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
1762                sizeof(get_resp.u.offload));
1763
1764         return 0;
1765 }
1766
1767 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
1768 {
1769         ena_com_handle_admin_completion(&ena_dev->admin_queue);
1770 }
1771
1772 /* ena_handle_specific_aenq_event:
1773  * return the handler that is relevant to the specific event group
1774  */
1775 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
1776                                                      u16 group)
1777 {
1778         struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
1779
1780         if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
1781                 return aenq_handlers->handlers[group];
1782
1783         return aenq_handlers->unimplemented_handler;
1784 }
1785
1786 /* ena_aenq_intr_handler:
1787  * handles the aenq incoming events.
1788  * pop events from the queue and apply the specific handler
1789  */
1790 void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
1791 {
1792         struct ena_admin_aenq_entry *aenq_e;
1793         struct ena_admin_aenq_common_desc *aenq_common;
1794         struct ena_com_aenq *aenq  = &dev->aenq;
1795         ena_aenq_handler handler_cb;
1796         u16 masked_head, processed = 0;
1797         u8 phase;
1798
1799         masked_head = aenq->head & (aenq->q_depth - 1);
1800         phase = aenq->phase;
1801         aenq_e = &aenq->entries[masked_head]; /* Get first entry */
1802         aenq_common = &aenq_e->aenq_common_desc;
1803
1804         /* Go over all the events */
1805         while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
1806                 phase) {
1807                 ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
1808                             aenq_common->group,
1809                             aenq_common->syndrom,
1810                             (unsigned long long)aenq_common->timestamp_low +
1811                             ((u64)aenq_common->timestamp_high << 32));
1812
1813                 /* Handle specific event*/
1814                 handler_cb = ena_com_get_specific_aenq_cb(dev,
1815                                                           aenq_common->group);
1816                 handler_cb(data, aenq_e); /* call the actual event handler*/
1817
1818                 /* Get next event entry */
1819                 masked_head++;
1820                 processed++;
1821
1822                 if (unlikely(masked_head == aenq->q_depth)) {
1823                         masked_head = 0;
1824                         phase = !phase;
1825                 }
1826                 aenq_e = &aenq->entries[masked_head];
1827                 aenq_common = &aenq_e->aenq_common_desc;
1828         }
1829
1830         aenq->head += processed;
1831         aenq->phase = phase;
1832
1833         /* Don't update aenq doorbell if there weren't any processed events */
1834         if (!processed)
1835                 return;
1836
1837         /* write the aenq doorbell after all AENQ descriptors were read */
1838         mb();
1839         ENA_REG_WRITE32((u32)aenq->head, (unsigned char *)dev->reg_bar
1840                         + ENA_REGS_AENQ_HEAD_DB_OFF);
1841 }
1842
1843 /* Sets the function Idx and Queue Idx to be used for
1844  * get full statistics feature
1845  */
1846 int ena_com_extended_stats_set_func_queue(struct ena_com_dev *ena_dev,
1847                                           u32 func_queue)
1848 {
1849         /* Function & Queue is acquired from user in the following format :
1850          * Bottom Half word:    funct
1851          * Top Half Word:       queue
1852          */
1853         ena_dev->stats_func = ENA_EXTENDED_STAT_GET_FUNCT(func_queue);
1854         ena_dev->stats_queue = ENA_EXTENDED_STAT_GET_QUEUE(func_queue);
1855
1856         return 0;
1857 }
1858
1859 int ena_com_dev_reset(struct ena_com_dev *ena_dev)
1860 {
1861         u32 stat, timeout, cap, reset_val;
1862         int rc;
1863
1864         stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1865         cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1866
1867         if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
1868                      (cap == ENA_MMIO_READ_TIMEOUT))) {
1869                 ena_trc_err("Reg read32 timeout occurred\n");
1870                 return ENA_COM_TIMER_EXPIRED;
1871         }
1872
1873         if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
1874                 ena_trc_err("Device isn't ready, can't reset device\n");
1875                 return ENA_COM_INVAL;
1876         }
1877
1878         timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
1879                         ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
1880         if (timeout == 0) {
1881                 ena_trc_err("Invalid timeout value\n");
1882                 return ENA_COM_INVAL;
1883         }
1884
1885         /* start reset */
1886         reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
1887         ENA_REG_WRITE32(reset_val, (unsigned char *)ena_dev->reg_bar
1888                         + ENA_REGS_DEV_CTL_OFF);
1889
1890         /* Write again the MMIO read request address */
1891         ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1892
1893         rc = wait_for_reset_state(ena_dev, timeout,
1894                                   ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
1895         if (rc != 0) {
1896                 ena_trc_err("Reset indication didn't turn on\n");
1897                 return rc;
1898         }
1899
1900         /* reset done */
1901         ENA_REG_WRITE32(0, (unsigned char *)ena_dev->reg_bar
1902                         + ENA_REGS_DEV_CTL_OFF);
1903         rc = wait_for_reset_state(ena_dev, timeout, 0);
1904         if (rc != 0) {
1905                 ena_trc_err("Reset indication didn't turn off\n");
1906                 return rc;
1907         }
1908
1909         return 0;
1910 }
1911
1912 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
1913                              struct ena_admin_aq_get_stats_cmd *get_cmd,
1914                              struct ena_admin_acq_get_stats_resp *get_resp,
1915                              enum ena_admin_get_stats_type type)
1916 {
1917         struct ena_com_admin_queue *admin_queue;
1918         int ret = 0;
1919
1920         if (!ena_dev) {
1921                 ena_trc_err("%s : ena_dev is NULL\n", __func__);
1922                 return ENA_COM_NO_DEVICE;
1923         }
1924
1925         admin_queue = &ena_dev->admin_queue;
1926
1927         get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
1928         get_cmd->aq_common_descriptor.flags = 0;
1929         get_cmd->type = type;
1930
1931         ret =  ena_com_execute_admin_command(
1932                         admin_queue,
1933                         (struct ena_admin_aq_entry *)get_cmd,
1934                         sizeof(*get_cmd),
1935                         (struct ena_admin_acq_entry *)get_resp,
1936                         sizeof(*get_resp));
1937
1938         if (unlikely(ret))
1939                 ena_trc_err("Failed to get stats. error: %d\n", ret);
1940
1941         return ret;
1942 }
1943
1944 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
1945                                 struct ena_admin_basic_stats *stats)
1946 {
1947         int ret = 0;
1948         struct ena_admin_aq_get_stats_cmd get_cmd;
1949         struct ena_admin_acq_get_stats_resp get_resp;
1950
1951         memset(&get_cmd, 0x0, sizeof(get_cmd));
1952         ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp,
1953                                 ENA_ADMIN_GET_STATS_TYPE_BASIC);
1954         if (likely(ret == 0))
1955                 memcpy(stats, &get_resp.basic_stats,
1956                        sizeof(get_resp.basic_stats));
1957
1958         return ret;
1959 }
1960
1961 int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff,
1962                                    u32 len)
1963 {
1964         int ret = 0;
1965         struct ena_admin_aq_get_stats_cmd get_cmd;
1966         struct ena_admin_acq_get_stats_resp get_resp;
1967         ena_mem_handle_t mem_handle = 0;
1968         void *virt_addr;
1969         dma_addr_t phys_addr;
1970
1971         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, len,
1972                                virt_addr, phys_addr, mem_handle);
1973         if (!virt_addr) {
1974                 ret = ENA_COM_NO_MEM;
1975                 goto done;
1976         }
1977         memset(&get_cmd, 0x0, sizeof(get_cmd));
1978         ret = ena_com_mem_addr_set(ena_dev,
1979                                    &get_cmd.u.control_buffer.address,
1980                                    phys_addr);
1981         if (unlikely(ret)) {
1982                 ena_trc_err("memory address set failed\n");
1983                 return ret;
1984         }
1985         get_cmd.u.control_buffer.length = len;
1986
1987         get_cmd.device_id = ena_dev->stats_func;
1988         get_cmd.queue_idx = ena_dev->stats_queue;
1989
1990         ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp,
1991                                 ENA_ADMIN_GET_STATS_TYPE_EXTENDED);
1992         if (ret < 0)
1993                 goto free_ext_stats_mem;
1994
1995         ret = snprintf(buff, len, "%s", (char *)virt_addr);
1996
1997 free_ext_stats_mem:
1998         ENA_MEM_FREE_COHERENT(ena_dev->dmadev, len, virt_addr, phys_addr,
1999                               mem_handle);
2000 done:
2001         return ret;
2002 }
2003
2004 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
2005 {
2006         struct ena_com_admin_queue *admin_queue;
2007         struct ena_admin_set_feat_cmd cmd;
2008         struct ena_admin_set_feat_resp resp;
2009         int ret = 0;
2010
2011         if (unlikely(!ena_dev)) {
2012                 ena_trc_err("%s : ena_dev is NULL\n", __func__);
2013                 return ENA_COM_NO_DEVICE;
2014         }
2015
2016         if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
2017                 ena_trc_info("Feature %d isn't supported\n", ENA_ADMIN_MTU);
2018                 return ENA_COM_PERMISSION;
2019         }
2020
2021         memset(&cmd, 0x0, sizeof(cmd));
2022         admin_queue = &ena_dev->admin_queue;
2023
2024         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2025         cmd.aq_common_descriptor.flags = 0;
2026         cmd.feat_common.feature_id = ENA_ADMIN_MTU;
2027         cmd.u.mtu.mtu = mtu;
2028
2029         ret = ena_com_execute_admin_command(admin_queue,
2030                                             (struct ena_admin_aq_entry *)&cmd,
2031                                             sizeof(cmd),
2032                                             (struct ena_admin_acq_entry *)&resp,
2033                                             sizeof(resp));
2034
2035         if (unlikely(ret)) {
2036                 ena_trc_err("Failed to set mtu %d. error: %d\n", mtu, ret);
2037                 return ENA_COM_INVAL;
2038         }
2039         return 0;
2040 }
2041
2042 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
2043                                  struct ena_admin_feature_offload_desc *offload)
2044 {
2045         int ret;
2046         struct ena_admin_get_feat_resp resp;
2047
2048         ret = ena_com_get_feature(ena_dev, &resp,
2049                                   ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
2050         if (unlikely(ret)) {
2051                 ena_trc_err("Failed to get offload capabilities %d\n", ret);
2052                 return ENA_COM_INVAL;
2053         }
2054
2055         memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
2056
2057         return 0;
2058 }
2059
2060 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
2061 {
2062         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2063         struct ena_rss *rss = &ena_dev->rss;
2064         struct ena_admin_set_feat_cmd cmd;
2065         struct ena_admin_set_feat_resp resp;
2066         struct ena_admin_get_feat_resp get_resp;
2067         int ret;
2068
2069         if (!ena_com_check_supported_feature_id(ena_dev,
2070                                                 ENA_ADMIN_RSS_HASH_FUNCTION)) {
2071                 ena_trc_info("Feature %d isn't supported\n",
2072                              ENA_ADMIN_RSS_HASH_FUNCTION);
2073                 return ENA_COM_PERMISSION;
2074         }
2075
2076         /* Validate hash function is supported */
2077         ret = ena_com_get_feature(ena_dev, &get_resp,
2078                                   ENA_ADMIN_RSS_HASH_FUNCTION);
2079         if (unlikely(ret))
2080                 return ret;
2081
2082         if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
2083                 ena_trc_err("Func hash %d isn't supported by device, abort\n",
2084                             rss->hash_func);
2085                 return ENA_COM_PERMISSION;
2086         }
2087
2088         memset(&cmd, 0x0, sizeof(cmd));
2089
2090         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2091         cmd.aq_common_descriptor.flags =
2092                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2093         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
2094         cmd.u.flow_hash_func.init_val = rss->hash_init_val;
2095         cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
2096
2097         ret = ena_com_mem_addr_set(ena_dev,
2098                                    &cmd.control_buffer.address,
2099                                    rss->hash_key_dma_addr);
2100         if (unlikely(ret)) {
2101                 ena_trc_err("memory address set failed\n");
2102                 return ret;
2103         }
2104
2105         cmd.control_buffer.length = sizeof(*rss->hash_key);
2106
2107         ret = ena_com_execute_admin_command(admin_queue,
2108                                             (struct ena_admin_aq_entry *)&cmd,
2109                                             sizeof(cmd),
2110                                             (struct ena_admin_acq_entry *)&resp,
2111                                             sizeof(resp));
2112         if (unlikely(ret)) {
2113                 ena_trc_err("Failed to set hash function %d. error: %d\n",
2114                             rss->hash_func, ret);
2115                 return ENA_COM_INVAL;
2116         }
2117
2118         return 0;
2119 }
2120
2121 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2122                                enum ena_admin_hash_functions func,
2123                                const u8 *key, u16 key_len, u32 init_val)
2124 {
2125         struct ena_rss *rss = &ena_dev->rss;
2126         struct ena_admin_get_feat_resp get_resp;
2127         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2128                 rss->hash_key;
2129         int rc;
2130
2131         /* Make sure size is a mult of DWs */
2132         if (unlikely(key_len & 0x3))
2133                 return ENA_COM_INVAL;
2134
2135         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2136                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2137                                     rss->hash_key_dma_addr,
2138                                     sizeof(*rss->hash_key));
2139         if (unlikely(rc))
2140                 return rc;
2141
2142         if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
2143                 ena_trc_err("Flow hash function %d isn't supported\n", func);
2144                 return ENA_COM_PERMISSION;
2145         }
2146
2147         switch (func) {
2148         case ENA_ADMIN_TOEPLITZ:
2149                 if (key_len > sizeof(hash_key->key)) {
2150                         ena_trc_err("key len (%hu) is bigger than the max supported (%zu)\n",
2151                                     key_len, sizeof(hash_key->key));
2152                         return ENA_COM_INVAL;
2153                 }
2154
2155                 memcpy(hash_key->key, key, key_len);
2156                 rss->hash_init_val = init_val;
2157                 hash_key->keys_num = key_len >> 2;
2158                 break;
2159         case ENA_ADMIN_CRC32:
2160                 rss->hash_init_val = init_val;
2161                 break;
2162         default:
2163                 ena_trc_err("Invalid hash function (%d)\n", func);
2164                 return ENA_COM_INVAL;
2165         }
2166
2167         rc = ena_com_set_hash_function(ena_dev);
2168
2169         /* Restore the old function */
2170         if (unlikely(rc))
2171                 ena_com_get_hash_function(ena_dev, NULL, NULL);
2172
2173         return rc;
2174 }
2175
2176 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2177                               enum ena_admin_hash_functions *func,
2178                               u8 *key)
2179 {
2180         struct ena_rss *rss = &ena_dev->rss;
2181         struct ena_admin_get_feat_resp get_resp;
2182         struct ena_admin_feature_rss_flow_hash_control *hash_key =
2183                 rss->hash_key;
2184         int rc;
2185
2186         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2187                                     ENA_ADMIN_RSS_HASH_FUNCTION,
2188                                     rss->hash_key_dma_addr,
2189                                     sizeof(*rss->hash_key));
2190         if (unlikely(rc))
2191                 return rc;
2192
2193         rss->hash_func = (enum ena_admin_hash_functions)get_resp.u.flow_hash_func.selected_func;
2194         if (func)
2195                 *func = rss->hash_func;
2196
2197         if (key)
2198                 memcpy(key, hash_key->key, hash_key->keys_num << 2);
2199
2200         return 0;
2201 }
2202
2203 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2204                           enum ena_admin_flow_hash_proto proto,
2205                           u16 *fields)
2206 {
2207         struct ena_rss *rss = &ena_dev->rss;
2208         struct ena_admin_get_feat_resp get_resp;
2209         int rc;
2210
2211         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2212                                     ENA_ADMIN_RSS_HASH_INPUT,
2213                                     rss->hash_ctrl_dma_addr,
2214                                     sizeof(*rss->hash_ctrl));
2215         if (unlikely(rc))
2216                 return rc;
2217
2218         if (fields)
2219                 *fields = rss->hash_ctrl->selected_fields[proto].fields;
2220
2221         return 0;
2222 }
2223
2224 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2225 {
2226         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2227         struct ena_rss *rss = &ena_dev->rss;
2228         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2229         struct ena_admin_set_feat_cmd cmd;
2230         struct ena_admin_set_feat_resp resp;
2231         int ret;
2232
2233         if (!ena_com_check_supported_feature_id(ena_dev,
2234                                                 ENA_ADMIN_RSS_HASH_INPUT)) {
2235                 ena_trc_info("Feature %d isn't supported\n",
2236                              ENA_ADMIN_RSS_HASH_INPUT);
2237                 return ENA_COM_PERMISSION;
2238         }
2239
2240         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2241         cmd.aq_common_descriptor.flags =
2242                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2243         cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2244         cmd.u.flow_hash_input.enabled_input_sort =
2245                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2246                 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2247
2248         ret = ena_com_mem_addr_set(ena_dev,
2249                                    &cmd.control_buffer.address,
2250                                    rss->hash_ctrl_dma_addr);
2251         if (unlikely(ret)) {
2252                 ena_trc_err("memory address set failed\n");
2253                 return ret;
2254         }
2255         cmd.control_buffer.length = sizeof(*hash_ctrl);
2256
2257         ret = ena_com_execute_admin_command(admin_queue,
2258                                             (struct ena_admin_aq_entry *)&cmd,
2259                                             sizeof(cmd),
2260                                             (struct ena_admin_acq_entry *)&resp,
2261                                             sizeof(resp));
2262         if (unlikely(ret)) {
2263                 ena_trc_err("Failed to set hash input. error: %d\n", ret);
2264                 ret = ENA_COM_INVAL;
2265         }
2266
2267         return 0;
2268 }
2269
2270 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2271 {
2272         struct ena_rss *rss = &ena_dev->rss;
2273         struct ena_admin_feature_rss_hash_control *hash_ctrl =
2274                 rss->hash_ctrl;
2275         u16 available_fields = 0;
2276         int rc, i;
2277
2278         /* Get the supported hash input */
2279         rc = ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
2280         if (unlikely(rc))
2281                 return rc;
2282
2283         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2284                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2285                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2286
2287         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2288                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2289                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2290
2291         hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2292                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2293                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2294
2295         hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2296                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2297                 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2298
2299         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2300                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2301
2302         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2303                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2304
2305         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2306                 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2307
2308         hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2309                 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2310
2311         for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2312                 available_fields = hash_ctrl->selected_fields[i].fields &
2313                                 hash_ctrl->supported_fields[i].fields;
2314                 if (available_fields != hash_ctrl->selected_fields[i].fields) {
2315                         ena_trc_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2316                                     i, hash_ctrl->supported_fields[i].fields,
2317                                     hash_ctrl->selected_fields[i].fields);
2318                         return ENA_COM_PERMISSION;
2319                 }
2320         }
2321
2322         rc = ena_com_set_hash_ctrl(ena_dev);
2323
2324         /* In case of failure, restore the old hash ctrl */
2325         if (unlikely(rc))
2326                 ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
2327
2328         return rc;
2329 }
2330
2331 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2332                            enum ena_admin_flow_hash_proto proto,
2333                            u16 hash_fields)
2334 {
2335         struct ena_rss *rss = &ena_dev->rss;
2336         struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2337         u16 supported_fields;
2338         int rc;
2339
2340         if (proto > ENA_ADMIN_RSS_PROTO_NUM) {
2341                 ena_trc_err("Invalid proto num (%u)\n", proto);
2342                 return ENA_COM_INVAL;
2343         }
2344
2345         /* Get the ctrl table */
2346         rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2347         if (unlikely(rc))
2348                 return rc;
2349
2350         /* Make sure all the fields are supported */
2351         supported_fields = hash_ctrl->supported_fields[proto].fields;
2352         if ((hash_fields & supported_fields) != hash_fields) {
2353                 ena_trc_err("proto %d doesn't support the required fields %x. supports only: %x\n",
2354                             proto, hash_fields, supported_fields);
2355         }
2356
2357         hash_ctrl->selected_fields[proto].fields = hash_fields;
2358
2359         rc = ena_com_set_hash_ctrl(ena_dev);
2360
2361         /* In case of failure, restore the old hash ctrl */
2362         if (unlikely(rc))
2363                 ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL);
2364
2365         return 0;
2366 }
2367
2368 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2369                                       u16 entry_idx, u16 entry_value)
2370 {
2371         struct ena_rss *rss = &ena_dev->rss;
2372
2373         if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2374                 return ENA_COM_INVAL;
2375
2376         if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2377                 return ENA_COM_INVAL;
2378
2379         rss->host_rss_ind_tbl[entry_idx] = entry_value;
2380
2381         return 0;
2382 }
2383
2384 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2385 {
2386         struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2387         struct ena_rss *rss = &ena_dev->rss;
2388         struct ena_admin_set_feat_cmd cmd;
2389         struct ena_admin_set_feat_resp resp;
2390         int ret = 0;
2391
2392         if (!ena_com_check_supported_feature_id(
2393                                 ena_dev,
2394                                 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
2395                 ena_trc_info("Feature %d isn't supported\n",
2396                              ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
2397                 return ENA_COM_PERMISSION;
2398         }
2399
2400         ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2401         if (ret) {
2402                 ena_trc_err("Failed to convert host indirection table to device table\n");
2403                 return ret;
2404         }
2405
2406         memset(&cmd, 0x0, sizeof(cmd));
2407
2408         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2409         cmd.aq_common_descriptor.flags =
2410                 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2411         cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
2412         cmd.u.ind_table.size = rss->tbl_log_size;
2413         cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2414
2415         ret = ena_com_mem_addr_set(ena_dev,
2416                                    &cmd.control_buffer.address,
2417                                    rss->rss_ind_tbl_dma_addr);
2418         if (unlikely(ret)) {
2419                 ena_trc_err("memory address set failed\n");
2420                 return ret;
2421         }
2422
2423         cmd.control_buffer.length = (1 << rss->tbl_log_size) *
2424                 sizeof(struct ena_admin_rss_ind_table_entry);
2425
2426         ret = ena_com_execute_admin_command(admin_queue,
2427                                             (struct ena_admin_aq_entry *)&cmd,
2428                                             sizeof(cmd),
2429                                             (struct ena_admin_acq_entry *)&resp,
2430                                             sizeof(resp));
2431
2432         if (unlikely(ret)) {
2433                 ena_trc_err("Failed to set indirect table. error: %d\n", ret);
2434                 return ENA_COM_INVAL;
2435         }
2436
2437         return 0;
2438 }
2439
2440 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2441 {
2442         struct ena_rss *rss = &ena_dev->rss;
2443         struct ena_admin_get_feat_resp get_resp;
2444         u32 tbl_size;
2445         int i, rc;
2446
2447         tbl_size = (1 << rss->tbl_log_size) *
2448                 sizeof(struct ena_admin_rss_ind_table_entry);
2449
2450         rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2451                                     ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
2452                                     rss->rss_ind_tbl_dma_addr,
2453                                     tbl_size);
2454         if (unlikely(rc))
2455                 return rc;
2456
2457         if (!ind_tbl)
2458                 return 0;
2459
2460         rc = ena_com_ind_tbl_convert_from_device(ena_dev);
2461         if (unlikely(rc))
2462                 return rc;
2463
2464         for (i = 0; i < (1 << rss->tbl_log_size); i++)
2465                 ind_tbl[i] = rss->host_rss_ind_tbl[i];
2466
2467         return 0;
2468 }
2469
2470 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2471 {
2472         int rc;
2473
2474         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2475
2476         rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2477         if (unlikely(rc))
2478                 goto err_indr_tbl;
2479
2480         rc = ena_com_hash_key_allocate(ena_dev);
2481         if (unlikely(rc))
2482                 goto err_hash_key;
2483
2484         rc = ena_com_hash_ctrl_init(ena_dev);
2485         if (unlikely(rc))
2486                 goto err_hash_ctrl;
2487
2488         return 0;
2489
2490 err_hash_ctrl:
2491         ena_com_hash_key_destroy(ena_dev);
2492 err_hash_key:
2493         ena_com_indirect_table_destroy(ena_dev);
2494 err_indr_tbl:
2495
2496         return rc;
2497 }
2498
2499 int ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2500 {
2501         ena_com_indirect_table_destroy(ena_dev);
2502         ena_com_hash_key_destroy(ena_dev);
2503         ena_com_hash_ctrl_destroy(ena_dev);
2504
2505         memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2506
2507         return 0;
2508 }
2509
2510 int ena_com_allocate_host_attribute(struct ena_com_dev *ena_dev,
2511                                     u32 debug_area_size)
2512 {
2513         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2514         int rc;
2515
2516         ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
2517                                SZ_4K,
2518                                host_attr->host_info,
2519                                host_attr->host_info_dma_addr,
2520                                host_attr->host_info_dma_handle);
2521         if (unlikely(!host_attr->host_info))
2522                 return ENA_COM_NO_MEM;
2523
2524         if (debug_area_size) {
2525                 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev,
2526                                        debug_area_size,
2527                                        host_attr->debug_area_virt_addr,
2528                                        host_attr->debug_area_dma_addr,
2529                                        host_attr->debug_area_dma_handle);
2530                 if (unlikely(!host_attr->debug_area_virt_addr)) {
2531                         rc = ENA_COM_NO_MEM;
2532                         goto err;
2533                 }
2534         }
2535
2536         host_attr->debug_area_size = debug_area_size;
2537
2538         return 0;
2539 err:
2540
2541         ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
2542                               SZ_4K,
2543                               host_attr->host_info,
2544                               host_attr->host_info_dma_addr,
2545                               host_attr->host_info_dma_handle);
2546         host_attr->host_info = NULL;
2547         return rc;
2548 }
2549
2550 void ena_com_delete_host_attribute(struct ena_com_dev *ena_dev)
2551 {
2552         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2553
2554         if (host_attr->host_info) {
2555                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
2556                                       SZ_4K,
2557                                       host_attr->host_info,
2558                                       host_attr->host_info_dma_addr,
2559                                       host_attr->host_info_dma_handle);
2560                 host_attr->host_info = NULL;
2561         }
2562
2563         if (host_attr->debug_area_virt_addr) {
2564                 ENA_MEM_FREE_COHERENT(ena_dev->dmadev,
2565                                       host_attr->debug_area_size,
2566                                       host_attr->debug_area_virt_addr,
2567                                       host_attr->debug_area_dma_addr,
2568                                       host_attr->debug_area_dma_handle);
2569                 host_attr->debug_area_virt_addr = NULL;
2570         }
2571 }
2572
2573 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2574 {
2575         struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2576         struct ena_com_admin_queue *admin_queue;
2577         struct ena_admin_set_feat_cmd cmd;
2578         struct ena_admin_set_feat_resp resp;
2579
2580         int ret = 0;
2581
2582         if (unlikely(!ena_dev)) {
2583                 ena_trc_err("%s : ena_dev is NULL\n", __func__);
2584                 return ENA_COM_NO_DEVICE;
2585         }
2586
2587         if (!ena_com_check_supported_feature_id(ena_dev,
2588                                                 ENA_ADMIN_HOST_ATTR_CONFIG)) {
2589                 ena_trc_warn("Set host attribute isn't supported\n");
2590                 return ENA_COM_PERMISSION;
2591         }
2592
2593         memset(&cmd, 0x0, sizeof(cmd));
2594         admin_queue = &ena_dev->admin_queue;
2595
2596         cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2597         cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2598
2599         ret = ena_com_mem_addr_set(ena_dev,
2600                                    &cmd.u.host_attr.debug_ba,
2601                                    host_attr->debug_area_dma_addr);
2602         if (unlikely(ret)) {
2603                 ena_trc_err("memory address set failed\n");
2604                 return ret;
2605         }
2606
2607         ret = ena_com_mem_addr_set(ena_dev,
2608                                    &cmd.u.host_attr.os_info_ba,
2609                                    host_attr->host_info_dma_addr);
2610         if (unlikely(ret)) {
2611                 ena_trc_err("memory address set failed\n");
2612                 return ret;
2613         }
2614
2615         cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2616
2617         ret = ena_com_execute_admin_command(admin_queue,
2618                                             (struct ena_admin_aq_entry *)&cmd,
2619                                             sizeof(cmd),
2620                                             (struct ena_admin_acq_entry *)&resp,
2621                                             sizeof(resp));
2622
2623         if (unlikely(ret))
2624                 ena_trc_err("Failed to set host attributes: %d\n", ret);
2625
2626         return ret;
2627 }
2628
2629 /* Interrupt moderation */
2630 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2631 {
2632         return ena_com_check_supported_feature_id(
2633                         ena_dev,
2634                         ENA_ADMIN_INTERRUPT_MODERATION);
2635 }
2636
2637 int
2638 ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2639                                                   u32 tx_coalesce_usecs)
2640 {
2641         if (!ena_dev->intr_delay_resolution) {
2642                 ena_trc_err("Illegal interrupt delay granularity value\n");
2643                 return ENA_COM_FAULT;
2644         }
2645
2646         ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
2647                 ena_dev->intr_delay_resolution;
2648
2649         return 0;
2650 }
2651
2652 int
2653 ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2654                                                   u32 rx_coalesce_usecs)
2655 {
2656         if (!ena_dev->intr_delay_resolution) {
2657                 ena_trc_err("Illegal interrupt delay granularity value\n");
2658                 return ENA_COM_FAULT;
2659         }
2660
2661         /* We use LOWEST entry of moderation table for storing
2662          * nonadaptive interrupt coalescing values
2663          */
2664         ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2665                 rx_coalesce_usecs / ena_dev->intr_delay_resolution;
2666
2667         return 0;
2668 }
2669
2670 void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
2671 {
2672         if (ena_dev->intr_moder_tbl)
2673                 ENA_MEM_FREE(ena_dev->dmadev, ena_dev->intr_moder_tbl);
2674         ena_dev->intr_moder_tbl = NULL;
2675 }
2676
2677 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2678 {
2679         struct ena_admin_get_feat_resp get_resp;
2680         u32 delay_resolution;
2681         int rc;
2682
2683         rc = ena_com_get_feature(ena_dev, &get_resp,
2684                                  ENA_ADMIN_INTERRUPT_MODERATION);
2685
2686         if (rc) {
2687                 if (rc == ENA_COM_PERMISSION) {
2688                         ena_trc_info("Feature %d isn't supported\n",
2689                                      ENA_ADMIN_INTERRUPT_MODERATION);
2690                         rc = 0;
2691                 } else {
2692                         ena_trc_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
2693                                     rc);
2694                 }
2695
2696                 /* no moderation supported, disable adaptive support */
2697                 ena_com_disable_adaptive_moderation(ena_dev);
2698                 return rc;
2699         }
2700
2701         rc = ena_com_init_interrupt_moderation_table(ena_dev);
2702         if (rc)
2703                 goto err;
2704
2705         /* if moderation is supported by device we set adaptive moderation */
2706         delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2707         ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2708         ena_com_enable_adaptive_moderation(ena_dev);
2709
2710         return 0;
2711 err:
2712         ena_com_destroy_interrupt_moderation(ena_dev);
2713         return rc;
2714 }
2715
2716 void
2717 ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
2718 {
2719         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2720
2721         if (!intr_moder_tbl)
2722                 return;
2723
2724         intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2725                 ENA_INTR_LOWEST_USECS;
2726         intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
2727                 ENA_INTR_LOWEST_PKTS;
2728         intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
2729                 ENA_INTR_LOWEST_BYTES;
2730
2731         intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
2732                 ENA_INTR_LOW_USECS;
2733         intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
2734                 ENA_INTR_LOW_PKTS;
2735         intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
2736                 ENA_INTR_LOW_BYTES;
2737
2738         intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
2739                 ENA_INTR_MID_USECS;
2740         intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
2741                 ENA_INTR_MID_PKTS;
2742         intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
2743                 ENA_INTR_MID_BYTES;
2744
2745         intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
2746                 ENA_INTR_HIGH_USECS;
2747         intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
2748                 ENA_INTR_HIGH_PKTS;
2749         intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
2750                 ENA_INTR_HIGH_BYTES;
2751
2752         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
2753                 ENA_INTR_HIGHEST_USECS;
2754         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
2755                 ENA_INTR_HIGHEST_PKTS;
2756         intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
2757                 ENA_INTR_HIGHEST_BYTES;
2758 }
2759
2760 unsigned int
2761 ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2762 {
2763         return ena_dev->intr_moder_tx_interval;
2764 }
2765
2766 unsigned int
2767 ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2768 {
2769         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2770
2771         if (intr_moder_tbl)
2772                 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
2773
2774         return 0;
2775 }
2776
2777 void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
2778                                         enum ena_intr_moder_level level,
2779                                         struct ena_intr_moder_entry *entry)
2780 {
2781         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2782
2783         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2784                 return;
2785
2786         intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
2787         if (ena_dev->intr_delay_resolution)
2788                 intr_moder_tbl[level].intr_moder_interval /=
2789                         ena_dev->intr_delay_resolution;
2790         intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
2791         intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
2792 }
2793
2794 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
2795                                        enum ena_intr_moder_level level,
2796                                        struct ena_intr_moder_entry *entry)
2797 {
2798         struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2799
2800         if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2801                 return;
2802
2803         entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
2804         if (ena_dev->intr_delay_resolution)
2805                 entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
2806         entry->pkts_per_interval =
2807         intr_moder_tbl[level].pkts_per_interval;
2808         entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
2809 }