drivers/net/liquidio/lio_rxtx.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Cavium, Inc. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_cycles.h>
  36 #include <rte_malloc.h>
  37
  38 #include "lio_logs.h"
  39 #include "lio_struct.h"
  40 #include "lio_ethdev.h"
  41 #include "lio_rxtx.h"
  42
  43 #define LIO_MAX_SG 12
  44 /* Flush iq if available tx_desc fall below LIO_FLUSH_WM */
  45 #define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2)
  46 #define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL
  47
  48 static void
  49 lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
  50 {
  51         uint32_t count = 0;
  52
  53         do {
  54                 count += droq->buffer_size;
  55         } while (count < LIO_MAX_RX_PKTLEN);
  56 }
  57
  58 static void
  59 lio_droq_reset_indices(struct lio_droq *droq)
  60 {
  61         droq->read_idx  = 0;
  62         droq->write_idx = 0;
  63         droq->refill_idx = 0;
  64         droq->refill_count = 0;
  65         rte_atomic64_set(&droq->pkts_pending, 0);
  66 }
  67
  68 static void
  69 lio_droq_destroy_ring_buffers(struct lio_droq *droq)
  70 {
  71         uint32_t i;
  72
  73         for (i = 0; i < droq->max_count; i++) {
  74                 if (droq->recv_buf_list[i].buffer) {
  75                         rte_pktmbuf_free((struct rte_mbuf *)
  76                                          droq->recv_buf_list[i].buffer);
  77                         droq->recv_buf_list[i].buffer = NULL;
  78                 }
  79         }
  80
  81         lio_droq_reset_indices(droq);
  82 }
  83
  84 static void *
  85 lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no)
  86 {
  87         struct lio_droq *droq = lio_dev->droq[q_no];
  88         struct rte_mempool *mpool = droq->mpool;
  89         struct rte_mbuf *m;
  90
  91         m = rte_pktmbuf_alloc(mpool);
  92         if (m == NULL) {
  93                 lio_dev_err(lio_dev, "Cannot allocate\n");
  94                 return NULL;
  95         }
  96
  97         rte_mbuf_refcnt_set(m, 1);
  98         m->next = NULL;
  99         m->data_off = RTE_PKTMBUF_HEADROOM;
 100         m->nb_segs = 1;
 101         m->pool = mpool;
 102
 103         return m;
 104 }
 105
 106 static int
 107 lio_droq_setup_ring_buffers(struct lio_device *lio_dev,
 108                             struct lio_droq *droq)
 109 {
 110         struct lio_droq_desc *desc_ring = droq->desc_ring;
 111         uint32_t i;
 112         void *buf;
 113
 114         for (i = 0; i < droq->max_count; i++) {
 115                 buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
 116                 if (buf == NULL) {
 117                         lio_dev_err(lio_dev, "buffer alloc failed\n");
 118                         droq->stats.rx_alloc_failure++;
 119                         lio_droq_destroy_ring_buffers(droq);
 120                         return -ENOMEM;
 121                 }
 122
 123                 droq->recv_buf_list[i].buffer = buf;
 124                 droq->info_list[i].length = 0;
 125
 126                 /* map ring buffers into memory */
 127                 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
 128                 desc_ring[i].buffer_ptr =
 129                         lio_map_ring(droq->recv_buf_list[i].buffer);
 130         }
 131
 132         lio_droq_reset_indices(droq);
 133
 134         lio_droq_compute_max_packet_bufs(droq);
 135
 136         return 0;
 137 }
 138
 139 static void
 140 lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz)
 141 {
 142         const struct rte_memzone *mz_tmp;
 143         int ret = 0;
 144
 145         if (mz == NULL) {
 146                 lio_dev_err(lio_dev, "Memzone NULL\n");
 147                 return;
 148         }
 149
 150         mz_tmp = rte_memzone_lookup(mz->name);
 151         if (mz_tmp == NULL) {
 152                 lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name);
 153                 return;
 154         }
 155
 156         ret = rte_memzone_free(mz);
 157         if (ret)
 158                 lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret);
 159 }
 160
 161 /**
 162  *  Frees the space for descriptor ring for the droq.
 163  *
 164  *  @param lio_dev      - pointer to the lio device structure
 165  *  @param q_no         - droq no.
 166  */
 167 static void
 168 lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no)
 169 {
 170         struct lio_droq *droq = lio_dev->droq[q_no];
 171
 172         lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no);
 173
 174         lio_droq_destroy_ring_buffers(droq);
 175         rte_free(droq->recv_buf_list);
 176         droq->recv_buf_list = NULL;
 177         lio_dma_zone_free(lio_dev, droq->info_mz);
 178         lio_dma_zone_free(lio_dev, droq->desc_ring_mz);
 179
 180         memset(droq, 0, LIO_DROQ_SIZE);
 181 }
 182
 183 static void *
 184 lio_alloc_info_buffer(struct lio_device *lio_dev,
 185                       struct lio_droq *droq, unsigned int socket_id)
 186 {
 187         droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
 188                                                  "info_list", droq->q_no,
 189                                                  (droq->max_count *
 190                                                         LIO_DROQ_INFO_SIZE),
 191                                                  RTE_CACHE_LINE_SIZE,
 192                                                  socket_id);
 193
 194         if (droq->info_mz == NULL)
 195                 return NULL;
 196
 197         droq->info_list_dma = droq->info_mz->phys_addr;
 198         droq->info_alloc_size = droq->info_mz->len;
 199         droq->info_base_addr = (size_t)droq->info_mz->addr;
 200
 201         return droq->info_mz->addr;
 202 }
 203
 204 /**
 205  *  Allocates space for the descriptor ring for the droq and
 206  *  sets the base addr, num desc etc in Octeon registers.
 207  *
 208  * @param lio_dev       - pointer to the lio device structure
 209  * @param q_no          - droq no.
 210  * @param app_ctx       - pointer to application context
 211  * @return Success: 0   Failure: -1
 212  */
 213 static int
 214 lio_init_droq(struct lio_device *lio_dev, uint32_t q_no,
 215               uint32_t num_descs, uint32_t desc_size,
 216               struct rte_mempool *mpool, unsigned int socket_id)
 217 {
 218         uint32_t c_refill_threshold;
 219         uint32_t desc_ring_size;
 220         struct lio_droq *droq;
 221
 222         lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no);
 223
 224         droq = lio_dev->droq[q_no];
 225         droq->lio_dev = lio_dev;
 226         droq->q_no = q_no;
 227         droq->mpool = mpool;
 228
 229         c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev);
 230
 231         droq->max_count = num_descs;
 232         droq->buffer_size = desc_size;
 233
 234         desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE;
 235         droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
 236                                                       "droq", q_no,
 237                                                       desc_ring_size,
 238                                                       RTE_CACHE_LINE_SIZE,
 239                                                       socket_id);
 240
 241         if (droq->desc_ring_mz == NULL) {
 242                 lio_dev_err(lio_dev,
 243                             "Output queue %d ring alloc failed\n", q_no);
 244                 return -1;
 245         }
 246
 247         droq->desc_ring_dma = droq->desc_ring_mz->phys_addr;
 248         droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr;
 249
 250         lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 251                     q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
 252         lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no,
 253                     droq->max_count);
 254
 255         droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id);
 256         if (droq->info_list == NULL) {
 257                 lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n");
 258                 goto init_droq_fail;
 259         }
 260
 261         droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
 262                                                  (droq->max_count *
 263                                                         LIO_DROQ_RECVBUF_SIZE),
 264                                                  RTE_CACHE_LINE_SIZE,
 265                                                  socket_id);
 266         if (droq->recv_buf_list == NULL) {
 267                 lio_dev_err(lio_dev,
 268                             "Output queue recv buf list alloc failed\n");
 269                 goto init_droq_fail;
 270         }
 271
 272         if (lio_droq_setup_ring_buffers(lio_dev, droq))
 273                 goto init_droq_fail;
 274
 275         droq->refill_threshold = c_refill_threshold;
 276
 277         rte_spinlock_init(&droq->lock);
 278
 279         lio_dev->fn_list.setup_oq_regs(lio_dev, q_no);
 280
 281         lio_dev->io_qmask.oq |= (1ULL << q_no);
 282
 283         return 0;
 284
 285 init_droq_fail:
 286         lio_delete_droq(lio_dev, q_no);
 287
 288         return -1;
 289 }
 290
 291 int
 292 lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs,
 293                int desc_size, struct rte_mempool *mpool, unsigned int socket_id)
 294 {
 295         struct lio_droq *droq;
 296
 297         PMD_INIT_FUNC_TRACE();
 298
 299         if (lio_dev->droq[oq_no]) {
 300                 lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no);
 301                 return 0;
 302         }
 303
 304         /* Allocate the DS for the new droq. */
 305         droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq),
 306                                   RTE_CACHE_LINE_SIZE, socket_id);
 307         if (droq == NULL)
 308                 return -ENOMEM;
 309
 310         lio_dev->droq[oq_no] = droq;
 311
 312         /* Initialize the Droq */
 313         if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool,
 314                           socket_id)) {
 315                 lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no);
 316                 rte_free(lio_dev->droq[oq_no]);
 317                 lio_dev->droq[oq_no] = NULL;
 318                 return -ENOMEM;
 319         }
 320
 321         lio_dev->num_oqs++;
 322
 323         lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs);
 324
 325         /* Send credit for octeon output queues. credits are always
 326          * sent after the output queue is enabled.
 327          */
 328         rte_write32(lio_dev->droq[oq_no]->max_count,
 329                     lio_dev->droq[oq_no]->pkts_credit_reg);
 330         rte_wmb();
 331
 332         return 0;
 333 }
 334
 335 static inline uint32_t
 336 lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len)
 337 {
 338         uint32_t buf_cnt = 0;
 339
 340         while (total_len > (buf_size * buf_cnt))
 341                 buf_cnt++;
 342
 343         return buf_cnt;
 344 }
 345
 346 /* If we were not able to refill all buffers, try to move around
 347  * the buffers that were not dispatched.
 348  */
 349 static inline uint32_t
 350 lio_droq_refill_pullup_descs(struct lio_droq *droq,
 351                              struct lio_droq_desc *desc_ring)
 352 {
 353         uint32_t refill_index = droq->refill_idx;
 354         uint32_t desc_refilled = 0;
 355
 356         while (refill_index != droq->read_idx) {
 357                 if (droq->recv_buf_list[refill_index].buffer) {
 358                         droq->recv_buf_list[droq->refill_idx].buffer =
 359                                 droq->recv_buf_list[refill_index].buffer;
 360                         desc_ring[droq->refill_idx].buffer_ptr =
 361                                 desc_ring[refill_index].buffer_ptr;
 362                         droq->recv_buf_list[refill_index].buffer = NULL;
 363                         desc_ring[refill_index].buffer_ptr = 0;
 364                         do {
 365                                 droq->refill_idx = lio_incr_index(
 366                                                         droq->refill_idx, 1,
 367                                                         droq->max_count);
 368                                 desc_refilled++;
 369                                 droq->refill_count--;
 370                         } while (droq->recv_buf_list[droq->refill_idx].buffer);
 371                 }
 372                 refill_index = lio_incr_index(refill_index, 1,
 373                                               droq->max_count);
 374         }       /* while */
 375
 376         return desc_refilled;
 377 }
 378
 379 /* lio_droq_refill
 380  *
 381  * @param lio_dev       - pointer to the lio device structure
 382  * @param droq          - droq in which descriptors require new buffers.
 383  *
 384  * Description:
 385  *  Called during normal DROQ processing in interrupt mode or by the poll
 386  *  thread to refill the descriptors from which buffers were dispatched
 387  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 388  *  up buffers (that were not dispatched) to form a contiguous ring.
 389  *
 390  * Returns:
 391  *  No of descriptors refilled.
 392  *
 393  * Locks:
 394  * This routine is called with droq->lock held.
 395  */
 396 static uint32_t
 397 lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq)
 398 {
 399         struct lio_droq_desc *desc_ring;
 400         uint32_t desc_refilled = 0;
 401         void *buf = NULL;
 402
 403         desc_ring = droq->desc_ring;
 404
 405         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 406                 /* If a valid buffer exists (happens if there is no dispatch),
 407                  * reuse the buffer, else allocate.
 408                  */
 409                 if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
 410                         buf = lio_recv_buffer_alloc(lio_dev, droq->q_no);
 411                         /* If a buffer could not be allocated, no point in
 412                          * continuing
 413                          */
 414                         if (buf == NULL) {
 415                                 droq->stats.rx_alloc_failure++;
 416                                 break;
 417                         }
 418
 419                         droq->recv_buf_list[droq->refill_idx].buffer = buf;
 420                 }
 421
 422                 desc_ring[droq->refill_idx].buffer_ptr =
 423                     lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer);
 424                 /* Reset any previous values in the length field. */
 425                 droq->info_list[droq->refill_idx].length = 0;
 426
 427                 droq->refill_idx = lio_incr_index(droq->refill_idx, 1,
 428                                                   droq->max_count);
 429                 desc_refilled++;
 430                 droq->refill_count--;
 431         }
 432
 433         if (droq->refill_count)
 434                 desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring);
 435
 436         /* if droq->refill_count
 437          * The refill count would not change in pass two. We only moved buffers
 438          * to close the gap in the ring, but we would still have the same no. of
 439          * buffers to refill.
 440          */
 441         return desc_refilled;
 442 }
 443
 444 static int
 445 lio_droq_fast_process_packet(struct lio_device *lio_dev,
 446                              struct lio_droq *droq,
 447                              struct rte_mbuf **rx_pkts)
 448 {
 449         struct rte_mbuf *nicbuf = NULL;
 450         struct lio_droq_info *info;
 451         uint32_t total_len = 0;
 452         int data_total_len = 0;
 453         uint32_t pkt_len = 0;
 454         union octeon_rh *rh;
 455         int data_pkts = 0;
 456
 457         info = &droq->info_list[droq->read_idx];
 458         lio_swap_8B_data((uint64_t *)info, 2);
 459
 460         if (!info->length)
 461                 return -1;
 462
 463         /* Len of resp hdr in included in the received data len. */
 464         info->length -= OCTEON_RH_SIZE;
 465         rh = &info->rh;
 466
 467         total_len += (uint32_t)info->length;
 468
 469         if (lio_opcode_slow_path(rh)) {
 470                 uint32_t buf_cnt;
 471
 472                 buf_cnt = lio_droq_get_bufcount(droq->buffer_size,
 473                                                 (uint32_t)info->length);
 474                 droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
 475                                                 droq->max_count);
 476                 droq->refill_count += buf_cnt;
 477         } else {
 478                 if (info->length <= droq->buffer_size) {
 479                         if (rh->r_dh.has_hash)
 480                                 pkt_len = (uint32_t)(info->length - 8);
 481                         else
 482                                 pkt_len = (uint32_t)info->length;
 483
 484                         nicbuf = droq->recv_buf_list[droq->read_idx].buffer;
 485                         droq->recv_buf_list[droq->read_idx].buffer = NULL;
 486                         droq->read_idx = lio_incr_index(
 487                                                 droq->read_idx, 1,
 488                                                 droq->max_count);
 489                         droq->refill_count++;
 490
 491                         if (likely(nicbuf != NULL)) {
 492                                 nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
 493                                 nicbuf->nb_segs = 1;
 494                                 nicbuf->next = NULL;
 495                                 /* We don't have a way to pass flags yet */
 496                                 nicbuf->ol_flags = 0;
 497                                 if (rh->r_dh.has_hash) {
 498                                         uint64_t *hash_ptr;
 499
 500                                         nicbuf->ol_flags |= PKT_RX_RSS_HASH;
 501                                         hash_ptr = rte_pktmbuf_mtod(nicbuf,
 502                                                                     uint64_t *);
 503                                         lio_swap_8B_data(hash_ptr, 1);
 504                                         nicbuf->hash.rss = (uint32_t)*hash_ptr;
 505                                         nicbuf->data_off += 8;
 506                                 }
 507
 508                                 nicbuf->pkt_len = pkt_len;
 509                                 nicbuf->data_len = pkt_len;
 510                                 nicbuf->port = lio_dev->port_id;
 511                                 /* Store the mbuf */
 512                                 rx_pkts[data_pkts++] = nicbuf;
 513                                 data_total_len += pkt_len;
 514                         }
 515
 516                         /* Prefetch buffer pointers when on a cache line
 517                          * boundary
 518                          */
 519                         if ((droq->read_idx & 3) == 0) {
 520                                 rte_prefetch0(
 521                                     &droq->recv_buf_list[droq->read_idx]);
 522                                 rte_prefetch0(
 523                                     &droq->info_list[droq->read_idx]);
 524                         }
 525                 } else {
 526                         struct rte_mbuf *first_buf = NULL;
 527                         struct rte_mbuf *last_buf = NULL;
 528
 529                         while (pkt_len < info->length) {
 530                                 int cpy_len = 0;
 531
 532                                 cpy_len = ((pkt_len + droq->buffer_size) >
 533                                                 info->length)
 534                                                 ? ((uint32_t)info->length -
 535                                                         pkt_len)
 536                                                 : droq->buffer_size;
 537
 538                                 nicbuf =
 539                                     droq->recv_buf_list[droq->read_idx].buffer;
 540                                 droq->recv_buf_list[droq->read_idx].buffer =
 541                                     NULL;
 542
 543                                 if (likely(nicbuf != NULL)) {
 544                                         /* Note the first seg */
 545                                         if (!pkt_len)
 546                                                 first_buf = nicbuf;
 547
 548                                         nicbuf->data_off = RTE_PKTMBUF_HEADROOM;
 549                                         nicbuf->nb_segs = 1;
 550                                         nicbuf->next = NULL;
 551                                         nicbuf->port = lio_dev->port_id;
 552                                         /* We don't have a way to pass
 553                                          * flags yet
 554                                          */
 555                                         nicbuf->ol_flags = 0;
 556                                         if ((!pkt_len) && (rh->r_dh.has_hash)) {
 557                                                 uint64_t *hash_ptr;
 558
 559                                                 nicbuf->ol_flags |=
 560                                                     PKT_RX_RSS_HASH;
 561                                                 hash_ptr = rte_pktmbuf_mtod(
 562                                                     nicbuf, uint64_t *);
 563                                                 lio_swap_8B_data(hash_ptr, 1);
 564                                                 nicbuf->hash.rss =
 565                                                     (uint32_t)*hash_ptr;
 566                                                 nicbuf->data_off += 8;
 567                                                 nicbuf->pkt_len = cpy_len - 8;
 568                                                 nicbuf->data_len = cpy_len - 8;
 569                                         } else {
 570                                                 nicbuf->pkt_len = cpy_len;
 571                                                 nicbuf->data_len = cpy_len;
 572                                         }
 573
 574                                         if (pkt_len)
 575                                                 first_buf->nb_segs++;
 576
 577                                         if (last_buf)
 578                                                 last_buf->next = nicbuf;
 579
 580                                         last_buf = nicbuf;
 581                                 } else {
 582                                         PMD_RX_LOG(lio_dev, ERR, "no buf\n");
 583                                 }
 584
 585                                 pkt_len += cpy_len;
 586                                 droq->read_idx = lio_incr_index(
 587                                                         droq->read_idx,
 588                                                         1, droq->max_count);
 589                                 droq->refill_count++;
 590
 591                                 /* Prefetch buffer pointers when on a
 592                                  * cache line boundary
 593                                  */
 594                                 if ((droq->read_idx & 3) == 0) {
 595                                         rte_prefetch0(&droq->recv_buf_list
 596                                                               [droq->read_idx]);
 597
 598                                         rte_prefetch0(
 599                                             &droq->info_list[droq->read_idx]);
 600                                 }
 601                         }
 602                         rx_pkts[data_pkts++] = first_buf;
 603                         if (rh->r_dh.has_hash)
 604                                 data_total_len += (pkt_len - 8);
 605                         else
 606                                 data_total_len += pkt_len;
 607                 }
 608
 609                 /* Inform upper layer about packet checksum verification */
 610                 struct rte_mbuf *m = rx_pkts[data_pkts - 1];
 611
 612                 if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED)
 613                         m->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 614
 615                 if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED)
 616                         m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 617         }
 618
 619         if (droq->refill_count >= droq->refill_threshold) {
 620                 int desc_refilled = lio_droq_refill(lio_dev, droq);
 621
 622                 /* Flush the droq descriptor data to memory to be sure
 623                  * that when we update the credits the data in memory is
 624                  * accurate.
 625                  */
 626                 rte_wmb();
 627                 rte_write32(desc_refilled, droq->pkts_credit_reg);
 628                 /* make sure mmio write completes */
 629                 rte_wmb();
 630         }
 631
 632         info->length = 0;
 633         info->rh.rh64 = 0;
 634
 635         droq->stats.pkts_received++;
 636         droq->stats.rx_pkts_received += data_pkts;
 637         droq->stats.rx_bytes_received += data_total_len;
 638         droq->stats.bytes_received += total_len;
 639
 640         return data_pkts;
 641 }
 642
 643 static uint32_t
 644 lio_droq_fast_process_packets(struct lio_device *lio_dev,
 645                               struct lio_droq *droq,
 646                               struct rte_mbuf **rx_pkts,
 647                               uint32_t pkts_to_process)
 648 {
 649         int ret, data_pkts = 0;
 650         uint32_t pkt;
 651
 652         for (pkt = 0; pkt < pkts_to_process; pkt++) {
 653                 ret = lio_droq_fast_process_packet(lio_dev, droq,
 654                                                    &rx_pkts[data_pkts]);
 655                 if (ret < 0) {
 656                         lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 657                                     lio_dev->port_id, droq->q_no,
 658                                     droq->read_idx, pkts_to_process);
 659                         break;
 660                 }
 661                 data_pkts += ret;
 662         }
 663
 664         rte_atomic64_sub(&droq->pkts_pending, pkt);
 665
 666         return data_pkts;
 667 }
 668
 669 static inline uint32_t
 670 lio_droq_check_hw_for_pkts(struct lio_droq *droq)
 671 {
 672         uint32_t last_count;
 673         uint32_t pkt_count;
 674
 675         pkt_count = rte_read32(droq->pkts_sent_reg);
 676
 677         last_count = pkt_count - droq->pkt_count;
 678         droq->pkt_count = pkt_count;
 679
 680         if (last_count)
 681                 rte_atomic64_add(&droq->pkts_pending, last_count);
 682
 683         return last_count;
 684 }
 685
 686 uint16_t
 687 lio_dev_recv_pkts(void *rx_queue,
 688                   struct rte_mbuf **rx_pkts,
 689                   uint16_t budget)
 690 {
 691         struct lio_droq *droq = rx_queue;
 692         struct lio_device *lio_dev = droq->lio_dev;
 693         uint32_t pkts_processed = 0;
 694         uint32_t pkt_count = 0;
 695
 696         lio_droq_check_hw_for_pkts(droq);
 697
 698         pkt_count = rte_atomic64_read(&droq->pkts_pending);
 699         if (!pkt_count)
 700                 return 0;
 701
 702         if (pkt_count > budget)
 703                 pkt_count = budget;
 704
 705         /* Grab the lock */
 706         rte_spinlock_lock(&droq->lock);
 707         pkts_processed = lio_droq_fast_process_packets(lio_dev,
 708                                                        droq, rx_pkts,
 709                                                        pkt_count);
 710
 711         if (droq->pkt_count) {
 712                 rte_write32(droq->pkt_count, droq->pkts_sent_reg);
 713                 droq->pkt_count = 0;
 714         }
 715
 716         /* Release the spin lock */
 717         rte_spinlock_unlock(&droq->lock);
 718
 719         return pkts_processed;
 720 }
 721
 722 void
 723 lio_delete_droq_queue(struct lio_device *lio_dev,
 724                       int oq_no)
 725 {
 726         lio_delete_droq(lio_dev, oq_no);
 727         lio_dev->num_oqs--;
 728         rte_free(lio_dev->droq[oq_no]);
 729         lio_dev->droq[oq_no] = NULL;
 730 }
 731
 732 /**
 733  *  lio_init_instr_queue()
 734  *  @param lio_dev      - pointer to the lio device structure.
 735  *  @param txpciq       - queue to be initialized.
 736  *
 737  *  Called at driver init time for each input queue. iq_conf has the
 738  *  configuration parameters for the queue.
 739  *
 740  *  @return  Success: 0 Failure: -1
 741  */
 742 static int
 743 lio_init_instr_queue(struct lio_device *lio_dev,
 744                      union octeon_txpciq txpciq,
 745                      uint32_t num_descs, unsigned int socket_id)
 746 {
 747         uint32_t iq_no = (uint32_t)txpciq.s.q_no;
 748         struct lio_instr_queue *iq;
 749         uint32_t instr_type;
 750         uint32_t q_size;
 751
 752         instr_type = LIO_IQ_INSTR_TYPE(lio_dev);
 753
 754         q_size = instr_type * num_descs;
 755         iq = lio_dev->instr_queue[iq_no];
 756         iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev,
 757                                              "instr_queue", iq_no, q_size,
 758                                              RTE_CACHE_LINE_SIZE,
 759                                              socket_id);
 760         if (iq->iq_mz == NULL) {
 761                 lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n",
 762                             iq_no);
 763                 return -1;
 764         }
 765
 766         iq->base_addr_dma = iq->iq_mz->phys_addr;
 767         iq->base_addr = (uint8_t *)iq->iq_mz->addr;
 768
 769         iq->max_count = num_descs;
 770
 771         /* Initialize a list to holds requests that have been posted to Octeon
 772          * but has yet to be fetched by octeon
 773          */
 774         iq->request_list = rte_zmalloc_socket("request_list",
 775                                               sizeof(*iq->request_list) *
 776                                                         num_descs,
 777                                               RTE_CACHE_LINE_SIZE,
 778                                               socket_id);
 779         if (iq->request_list == NULL) {
 780                 lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n",
 781                             iq_no);
 782                 lio_dma_zone_free(lio_dev, iq->iq_mz);
 783                 return -1;
 784         }
 785
 786         lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n",
 787                     iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
 788                     iq->max_count);
 789
 790         iq->lio_dev = lio_dev;
 791         iq->txpciq.txpciq64 = txpciq.txpciq64;
 792         iq->fill_cnt = 0;
 793         iq->host_write_index = 0;
 794         iq->lio_read_index = 0;
 795         iq->flush_index = 0;
 796
 797         rte_atomic64_set(&iq->instr_pending, 0);
 798
 799         /* Initialize the spinlock for this instruction queue */
 800         rte_spinlock_init(&iq->lock);
 801         rte_spinlock_init(&iq->post_lock);
 802
 803         rte_atomic64_clear(&iq->iq_flush_running);
 804
 805         lio_dev->io_qmask.iq |= (1ULL << iq_no);
 806
 807         /* Set the 32B/64B mode for each input queue */
 808         lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no);
 809         iq->iqcmd_64B = (instr_type == 64);
 810
 811         lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no);
 812
 813         return 0;
 814 }
 815
 816 int
 817 lio_setup_instr_queue0(struct lio_device *lio_dev)
 818 {
 819         union octeon_txpciq txpciq;
 820         uint32_t num_descs = 0;
 821         uint32_t iq_no = 0;
 822
 823         num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev);
 824
 825         lio_dev->num_iqs = 0;
 826
 827         lio_dev->instr_queue[0] = rte_zmalloc(NULL,
 828                                         sizeof(struct lio_instr_queue), 0);
 829         if (lio_dev->instr_queue[0] == NULL)
 830                 return -ENOMEM;
 831
 832         lio_dev->instr_queue[0]->q_index = 0;
 833         lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0;
 834         txpciq.txpciq64 = 0;
 835         txpciq.s.q_no = iq_no;
 836         txpciq.s.pkind = lio_dev->pfvf_hsword.pkind;
 837         txpciq.s.use_qpg = 0;
 838         txpciq.s.qpg = 0;
 839         if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) {
 840                 rte_free(lio_dev->instr_queue[0]);
 841                 lio_dev->instr_queue[0] = NULL;
 842                 return -1;
 843         }
 844
 845         lio_dev->num_iqs++;
 846
 847         return 0;
 848 }
 849
 850 /**
 851  *  lio_delete_instr_queue()
 852  *  @param lio_dev      - pointer to the lio device structure.
 853  *  @param iq_no        - queue to be deleted.
 854  *
 855  *  Called at driver unload time for each input queue. Deletes all
 856  *  allocated resources for the input queue.
 857  */
 858 static void
 859 lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no)
 860 {
 861         struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
 862
 863         rte_free(iq->request_list);
 864         iq->request_list = NULL;
 865         lio_dma_zone_free(lio_dev, iq->iq_mz);
 866 }
 867
 868 void
 869 lio_free_instr_queue0(struct lio_device *lio_dev)
 870 {
 871         lio_delete_instr_queue(lio_dev, 0);
 872         rte_free(lio_dev->instr_queue[0]);
 873         lio_dev->instr_queue[0] = NULL;
 874         lio_dev->num_iqs--;
 875 }
 876
 877 /* Return 0 on success, -1 on failure */
 878 int
 879 lio_setup_iq(struct lio_device *lio_dev, int q_index,
 880              union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx,
 881              unsigned int socket_id)
 882 {
 883         uint32_t iq_no = (uint32_t)txpciq.s.q_no;
 884
 885         if (lio_dev->instr_queue[iq_no]) {
 886                 lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n",
 887                             iq_no);
 888                 lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64;
 889                 lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
 890                 return 0;
 891         }
 892
 893         lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue",
 894                                                 sizeof(struct lio_instr_queue),
 895                                                 RTE_CACHE_LINE_SIZE, socket_id);
 896         if (lio_dev->instr_queue[iq_no] == NULL)
 897                 return -1;
 898
 899         lio_dev->instr_queue[iq_no]->q_index = q_index;
 900         lio_dev->instr_queue[iq_no]->app_ctx = app_ctx;
 901
 902         if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id))
 903                 goto release_lio_iq;
 904
 905         lio_dev->num_iqs++;
 906         if (lio_dev->fn_list.enable_io_queues(lio_dev))
 907                 goto delete_lio_iq;
 908
 909         return 0;
 910
 911 delete_lio_iq:
 912         lio_delete_instr_queue(lio_dev, iq_no);
 913         lio_dev->num_iqs--;
 914 release_lio_iq:
 915         rte_free(lio_dev->instr_queue[iq_no]);
 916         lio_dev->instr_queue[iq_no] = NULL;
 917
 918         return -1;
 919 }
 920
 921 int
 922 lio_wait_for_instr_fetch(struct lio_device *lio_dev)
 923 {
 924         int pending, instr_cnt;
 925         int i, retry = 1000;
 926
 927         do {
 928                 instr_cnt = 0;
 929
 930                 for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) {
 931                         if (!(lio_dev->io_qmask.iq & (1ULL << i)))
 932                                 continue;
 933
 934                         if (lio_dev->instr_queue[i] == NULL)
 935                                 break;
 936
 937                         pending = rte_atomic64_read(
 938                             &lio_dev->instr_queue[i]->instr_pending);
 939                         if (pending)
 940                                 lio_flush_iq(lio_dev, lio_dev->instr_queue[i]);
 941
 942                         instr_cnt += pending;
 943                 }
 944
 945                 if (instr_cnt == 0)
 946                         break;
 947
 948                 rte_delay_ms(1);
 949
 950         } while (retry-- && instr_cnt);
 951
 952         return instr_cnt;
 953 }
 954
 955 static inline void
 956 lio_ring_doorbell(struct lio_device *lio_dev,
 957                   struct lio_instr_queue *iq)
 958 {
 959         if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) {
 960                 rte_write32(iq->fill_cnt, iq->doorbell_reg);
 961                 /* make sure doorbell write goes through */
 962                 rte_wmb();
 963                 iq->fill_cnt = 0;
 964         }
 965 }
 966
 967 static inline void
 968 copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd)
 969 {
 970         uint8_t *iqptr, cmdsize;
 971
 972         cmdsize = ((iq->iqcmd_64B) ? 64 : 32);
 973         iqptr = iq->base_addr + (cmdsize * iq->host_write_index);
 974
 975         rte_memcpy(iqptr, cmd, cmdsize);
 976 }
 977
 978 static inline struct lio_iq_post_status
 979 post_command2(struct lio_instr_queue *iq, uint8_t *cmd)
 980 {
 981         struct lio_iq_post_status st;
 982
 983         st.status = LIO_IQ_SEND_OK;
 984
 985         /* This ensures that the read index does not wrap around to the same
 986          * position if queue gets full before Octeon could fetch any instr.
 987          */
 988         if (rte_atomic64_read(&iq->instr_pending) >=
 989                         (int32_t)(iq->max_count - 1)) {
 990                 st.status = LIO_IQ_SEND_FAILED;
 991                 st.index = -1;
 992                 return st;
 993         }
 994
 995         if (rte_atomic64_read(&iq->instr_pending) >=
 996                         (int32_t)(iq->max_count - 2))
 997                 st.status = LIO_IQ_SEND_STOP;
 998
 999         copy_cmd_into_iq(iq, cmd);
1000
1001         /* "index" is returned, host_write_index is modified. */
1002         st.index = iq->host_write_index;
1003         iq->host_write_index = lio_incr_index(iq->host_write_index, 1,
1004                                               iq->max_count);
1005         iq->fill_cnt++;
1006
1007         /* Flush the command into memory. We need to be sure the data is in
1008          * memory before indicating that the instruction is pending.
1009          */
1010         rte_wmb();
1011
1012         rte_atomic64_inc(&iq->instr_pending);
1013
1014         return st;
1015 }
1016
1017 static inline void
1018 lio_add_to_request_list(struct lio_instr_queue *iq,
1019                         int idx, void *buf, int reqtype)
1020 {
1021         iq->request_list[idx].buf = buf;
1022         iq->request_list[idx].reqtype = reqtype;
1023 }
1024
1025 static inline void
1026 lio_free_netsgbuf(void *buf)
1027 {
1028         struct lio_buf_free_info *finfo = buf;
1029         struct lio_device *lio_dev = finfo->lio_dev;
1030         struct rte_mbuf *m = finfo->mbuf;
1031         struct lio_gather *g = finfo->g;
1032         uint8_t iq = finfo->iq_no;
1033
1034         /* This will take care of multiple segments also */
1035         rte_pktmbuf_free(m);
1036
1037         rte_spinlock_lock(&lio_dev->glist_lock[iq]);
1038         STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries);
1039         rte_spinlock_unlock(&lio_dev->glist_lock[iq]);
1040         rte_free(finfo);
1041 }
1042
1043 /* Can only run in process context */
1044 static int
1045 lio_process_iq_request_list(struct lio_device *lio_dev,
1046                             struct lio_instr_queue *iq)
1047 {
1048         struct octeon_instr_irh *irh = NULL;
1049         uint32_t old = iq->flush_index;
1050         struct lio_soft_command *sc;
1051         uint32_t inst_count = 0;
1052         int reqtype;
1053         void *buf;
1054
1055         while (old != iq->lio_read_index) {
1056                 reqtype = iq->request_list[old].reqtype;
1057                 buf     = iq->request_list[old].buf;
1058
1059                 if (reqtype == LIO_REQTYPE_NONE)
1060                         goto skip_this;
1061
1062                 switch (reqtype) {
1063                 case LIO_REQTYPE_NORESP_NET:
1064                         rte_pktmbuf_free((struct rte_mbuf *)buf);
1065                         break;
1066                 case LIO_REQTYPE_NORESP_NET_SG:
1067                         lio_free_netsgbuf(buf);
1068                         break;
1069                 case LIO_REQTYPE_SOFT_COMMAND:
1070                         sc = buf;
1071                         irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1072                         if (irh->rflag) {
1073                                 /* We're expecting a response from Octeon.
1074                                  * It's up to lio_process_ordered_list() to
1075                                  * process sc. Add sc to the ordered soft
1076                                  * command response list because we expect
1077                                  * a response from Octeon.
1078                                  */
1079                                 rte_spinlock_lock(&lio_dev->response_list.lock);
1080                                 rte_atomic64_inc(
1081                                     &lio_dev->response_list.pending_req_count);
1082                                 STAILQ_INSERT_TAIL(
1083                                         &lio_dev->response_list.head,
1084                                         &sc->node, entries);
1085                                 rte_spinlock_unlock(
1086                                                 &lio_dev->response_list.lock);
1087                         } else {
1088                                 if (sc->callback) {
1089                                         /* This callback must not sleep */
1090                                         sc->callback(LIO_REQUEST_DONE,
1091                                                      sc->callback_arg);
1092                                 }
1093                         }
1094                         break;
1095                 default:
1096                         lio_dev_err(lio_dev,
1097                                     "Unknown reqtype: %d buf: %p at idx %d\n",
1098                                     reqtype, buf, old);
1099                 }
1100
1101                 iq->request_list[old].buf = NULL;
1102                 iq->request_list[old].reqtype = 0;
1103
1104 skip_this:
1105                 inst_count++;
1106                 old = lio_incr_index(old, 1, iq->max_count);
1107         }
1108
1109         iq->flush_index = old;
1110
1111         return inst_count;
1112 }
1113
1114 static void
1115 lio_update_read_index(struct lio_instr_queue *iq)
1116 {
1117         uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg);
1118         uint32_t last_done;
1119
1120         last_done = pkt_in_done - iq->pkt_in_done;
1121         iq->pkt_in_done = pkt_in_done;
1122
1123         /* Add last_done and modulo with the IQ size to get new index */
1124         iq->lio_read_index = (iq->lio_read_index +
1125                         (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) %
1126                         iq->max_count;
1127 }
1128
1129 int
1130 lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq)
1131 {
1132         uint32_t tot_inst_processed = 0;
1133         uint32_t inst_processed = 0;
1134         int tx_done = 1;
1135
1136         if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0)
1137                 return tx_done;
1138
1139         rte_spinlock_lock(&iq->lock);
1140
1141         lio_update_read_index(iq);
1142
1143         do {
1144                 /* Process any outstanding IQ packets. */
1145                 if (iq->flush_index == iq->lio_read_index)
1146                         break;
1147
1148                 inst_processed = lio_process_iq_request_list(lio_dev, iq);
1149
1150                 if (inst_processed) {
1151                         rte_atomic64_sub(&iq->instr_pending, inst_processed);
1152                         iq->stats.instr_processed += inst_processed;
1153                 }
1154
1155                 tot_inst_processed += inst_processed;
1156                 inst_processed = 0;
1157
1158         } while (1);
1159
1160         rte_spinlock_unlock(&iq->lock);
1161
1162         rte_atomic64_clear(&iq->iq_flush_running);
1163
1164         return tx_done;
1165 }
1166
1167 static int
1168 lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd,
1169                  void *buf, uint32_t datasize, uint32_t reqtype)
1170 {
1171         struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
1172         struct lio_iq_post_status st;
1173
1174         rte_spinlock_lock(&iq->post_lock);
1175
1176         st = post_command2(iq, cmd);
1177
1178         if (st.status != LIO_IQ_SEND_FAILED) {
1179                 lio_add_to_request_list(iq, st.index, buf, reqtype);
1180                 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent,
1181                                               datasize);
1182                 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1);
1183
1184                 lio_ring_doorbell(lio_dev, iq);
1185         } else {
1186                 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1);
1187         }
1188
1189         rte_spinlock_unlock(&iq->post_lock);
1190
1191         return st.status;
1192 }
1193
1194 void
1195 lio_prepare_soft_command(struct lio_device *lio_dev,
1196                          struct lio_soft_command *sc, uint8_t opcode,
1197                          uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0,
1198                          uint64_t ossp1)
1199 {
1200         struct octeon_instr_pki_ih3 *pki_ih3;
1201         struct octeon_instr_ih3 *ih3;
1202         struct octeon_instr_irh *irh;
1203         struct octeon_instr_rdp *rdp;
1204
1205         RTE_ASSERT(opcode <= 15);
1206         RTE_ASSERT(subcode <= 127);
1207
1208         ih3       = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
1209
1210         ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind;
1211
1212         pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3;
1213
1214         pki_ih3->w      = 1;
1215         pki_ih3->raw    = 1;
1216         pki_ih3->utag   = 1;
1217         pki_ih3->uqpg   = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg;
1218         pki_ih3->utt    = 1;
1219
1220         pki_ih3->tag    = LIO_CONTROL;
1221         pki_ih3->tagtype = OCTEON_ATOMIC_TAG;
1222         pki_ih3->qpg    = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg;
1223         pki_ih3->pm     = 0x7;
1224         pki_ih3->sl     = 8;
1225
1226         if (sc->datasize)
1227                 ih3->dlengsz = sc->datasize;
1228
1229         irh             = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1230         irh->opcode     = opcode;
1231         irh->subcode    = subcode;
1232
1233         /* opcode/subcode specific parameters (ossp) */
1234         irh->ossp = irh_ossp;
1235         sc->cmd.cmd3.ossp[0] = ossp0;
1236         sc->cmd.cmd3.ossp[1] = ossp1;
1237
1238         if (sc->rdatasize) {
1239                 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp;
1240                 rdp->pcie_port = lio_dev->pcie_port;
1241                 rdp->rlen      = sc->rdatasize;
1242                 irh->rflag = 1;
1243                 /* PKI IH3 */
1244                 ih3->fsz    = OCTEON_SOFT_CMD_RESP_IH3;
1245         } else {
1246                 irh->rflag = 0;
1247                 /* PKI IH3 */
1248                 ih3->fsz    = OCTEON_PCI_CMD_O3;
1249         }
1250 }
1251
1252 int
1253 lio_send_soft_command(struct lio_device *lio_dev,
1254                       struct lio_soft_command *sc)
1255 {
1256         struct octeon_instr_ih3 *ih3;
1257         struct octeon_instr_irh *irh;
1258         uint32_t len = 0;
1259
1260         ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3;
1261         if (ih3->dlengsz) {
1262                 RTE_ASSERT(sc->dmadptr);
1263                 sc->cmd.cmd3.dptr = sc->dmadptr;
1264         }
1265
1266         irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh;
1267         if (irh->rflag) {
1268                 RTE_ASSERT(sc->dmarptr);
1269                 RTE_ASSERT(sc->status_word != NULL);
1270                 *sc->status_word = LIO_COMPLETION_WORD_INIT;
1271                 sc->cmd.cmd3.rptr = sc->dmarptr;
1272         }
1273
1274         len = (uint32_t)ih3->dlengsz;
1275
1276         if (sc->wait_time)
1277                 sc->timeout = lio_uptime + sc->wait_time;
1278
1279         return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len,
1280                                 LIO_REQTYPE_SOFT_COMMAND);
1281 }
1282
1283 int
1284 lio_setup_sc_buffer_pool(struct lio_device *lio_dev)
1285 {
1286         char sc_pool_name[RTE_MEMPOOL_NAMESIZE];
1287         uint16_t buf_size;
1288
1289         buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM;
1290         snprintf(sc_pool_name, sizeof(sc_pool_name),
1291                  "lio_sc_pool_%u", lio_dev->port_id);
1292         lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name,
1293                                                 LIO_MAX_SOFT_COMMAND_BUFFERS,
1294                                                 0, 0, buf_size, SOCKET_ID_ANY);
1295         return 0;
1296 }
1297
1298 void
1299 lio_free_sc_buffer_pool(struct lio_device *lio_dev)
1300 {
1301         rte_mempool_free(lio_dev->sc_buf_pool);
1302 }
1303
1304 struct lio_soft_command *
1305 lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize,
1306                        uint32_t rdatasize, uint32_t ctxsize)
1307 {
1308         uint32_t offset = sizeof(struct lio_soft_command);
1309         struct lio_soft_command *sc;
1310         struct rte_mbuf *m;
1311         uint64_t dma_addr;
1312
1313         RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <=
1314                    LIO_SOFT_COMMAND_BUFFER_SIZE);
1315
1316         m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool);
1317         if (m == NULL) {
1318                 lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n");
1319                 return NULL;
1320         }
1321
1322         /* set rte_mbuf data size and there is only 1 segment */
1323         m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE;
1324         m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE;
1325
1326         /* use rte_mbuf buffer for soft command */
1327         sc = rte_pktmbuf_mtod(m, struct lio_soft_command *);
1328         memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE);
1329         sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE;
1330         sc->dma_addr = rte_mbuf_data_dma_addr(m);
1331         sc->mbuf = m;
1332
1333         dma_addr = sc->dma_addr;
1334
1335         if (ctxsize) {
1336                 sc->ctxptr = (uint8_t *)sc + offset;
1337                 sc->ctxsize = ctxsize;
1338         }
1339
1340         /* Start data at 128 byte boundary */
1341         offset = (offset + ctxsize + 127) & 0xffffff80;
1342
1343         if (datasize) {
1344                 sc->virtdptr = (uint8_t *)sc + offset;
1345                 sc->dmadptr = dma_addr + offset;
1346                 sc->datasize = datasize;
1347         }
1348
1349         /* Start rdata at 128 byte boundary */
1350         offset = (offset + datasize + 127) & 0xffffff80;
1351
1352         if (rdatasize) {
1353                 RTE_ASSERT(rdatasize >= 16);
1354                 sc->virtrptr = (uint8_t *)sc + offset;
1355                 sc->dmarptr = dma_addr + offset;
1356                 sc->rdatasize = rdatasize;
1357                 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) +
1358                                                rdatasize - 8);
1359         }
1360
1361         return sc;
1362 }
1363
1364 void
1365 lio_free_soft_command(struct lio_soft_command *sc)
1366 {
1367         rte_pktmbuf_free(sc->mbuf);
1368 }
1369
1370 void
1371 lio_setup_response_list(struct lio_device *lio_dev)
1372 {
1373         STAILQ_INIT(&lio_dev->response_list.head);
1374         rte_spinlock_init(&lio_dev->response_list.lock);
1375         rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0);
1376 }
1377
1378 int
1379 lio_process_ordered_list(struct lio_device *lio_dev)
1380 {
1381         int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS;
1382         struct lio_response_list *ordered_sc_list;
1383         struct lio_soft_command *sc;
1384         int request_complete = 0;
1385         uint64_t status64;
1386         uint32_t status;
1387
1388         ordered_sc_list = &lio_dev->response_list;
1389
1390         do {
1391                 rte_spinlock_lock(&ordered_sc_list->lock);
1392
1393                 if (STAILQ_EMPTY(&ordered_sc_list->head)) {
1394                         /* ordered_sc_list is empty; there is
1395                          * nothing to process
1396                          */
1397                         rte_spinlock_unlock(&ordered_sc_list->lock);
1398                         return -1;
1399                 }
1400
1401                 sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head,
1402                                              struct lio_soft_command, node);
1403
1404                 status = LIO_REQUEST_PENDING;
1405
1406                 /* check if octeon has finished DMA'ing a response
1407                  * to where rptr is pointing to
1408                  */
1409                 status64 = *sc->status_word;
1410
1411                 if (status64 != LIO_COMPLETION_WORD_INIT) {
1412                         /* This logic ensures that all 64b have been written.
1413                          * 1. check byte 0 for non-FF
1414                          * 2. if non-FF, then swap result from BE to host order
1415                          * 3. check byte 7 (swapped to 0) for non-FF
1416                          * 4. if non-FF, use the low 32-bit status code
1417                          * 5. if either byte 0 or byte 7 is FF, don't use status
1418                          */
1419                         if ((status64 & 0xff) != 0xff) {
1420                                 lio_swap_8B_data(&status64, 1);
1421                                 if (((status64 & 0xff) != 0xff)) {
1422                                         /* retrieve 16-bit firmware status */
1423                                         status = (uint32_t)(status64 &
1424                                                             0xffffULL);
1425                                         if (status) {
1426                                                 status =
1427                                                 LIO_FIRMWARE_STATUS_CODE(
1428                                                                         status);
1429                                         } else {
1430                                                 /* i.e. no error */
1431                                                 status = LIO_REQUEST_DONE;
1432                                         }
1433                                 }
1434                         }
1435                 } else if ((sc->timeout && lio_check_timeout(lio_uptime,
1436                                                              sc->timeout))) {
1437                         lio_dev_err(lio_dev,
1438                                     "cmd failed, timeout (%ld, %ld)\n",
1439                                     (long)lio_uptime, (long)sc->timeout);
1440                         status = LIO_REQUEST_TIMEOUT;
1441                 }
1442
1443                 if (status != LIO_REQUEST_PENDING) {
1444                         /* we have received a response or we have timed out.
1445                          * remove node from linked list
1446                          */
1447                         STAILQ_REMOVE(&ordered_sc_list->head,
1448                                       &sc->node, lio_stailq_node, entries);
1449                         rte_atomic64_dec(
1450                             &lio_dev->response_list.pending_req_count);
1451                         rte_spinlock_unlock(&ordered_sc_list->lock);
1452
1453                         if (sc->callback)
1454                                 sc->callback(status, sc->callback_arg);
1455
1456                         request_complete++;
1457                 } else {
1458                         /* no response yet */
1459                         request_complete = 0;
1460                         rte_spinlock_unlock(&ordered_sc_list->lock);
1461                 }
1462
1463                 /* If we hit the Max Ordered requests to process every loop,
1464                  * we quit and let this function be invoked the next time
1465                  * the poll thread runs to process the remaining requests.
1466                  * This function can take up the entire CPU if there is
1467                  * no upper limit to the requests processed.
1468                  */
1469                 if (request_complete >= resp_to_process)
1470                         break;
1471         } while (request_complete);
1472
1473         return 0;
1474 }
1475
1476 static inline struct lio_stailq_node *
1477 list_delete_first_node(struct lio_stailq_head *head)
1478 {
1479         struct lio_stailq_node *node;
1480
1481         if (STAILQ_EMPTY(head))
1482                 node = NULL;
1483         else
1484                 node = STAILQ_FIRST(head);
1485
1486         if (node)
1487                 STAILQ_REMOVE(head, node, lio_stailq_node, entries);
1488
1489         return node;
1490 }
1491
1492 void
1493 lio_delete_sglist(struct lio_instr_queue *txq)
1494 {
1495         struct lio_device *lio_dev = txq->lio_dev;
1496         int iq_no = txq->q_index;
1497         struct lio_gather *g;
1498
1499         if (lio_dev->glist_head == NULL)
1500                 return;
1501
1502         do {
1503                 g = (struct lio_gather *)list_delete_first_node(
1504                                                 &lio_dev->glist_head[iq_no]);
1505                 if (g) {
1506                         if (g->sg)
1507                                 rte_free(
1508                                     (void *)((unsigned long)g->sg - g->adjust));
1509                         rte_free(g);
1510                 }
1511         } while (g);
1512 }
1513
1514 /**
1515  * \brief Setup gather lists
1516  * @param lio per-network private data
1517  */
1518 int
1519 lio_setup_sglists(struct lio_device *lio_dev, int iq_no,
1520                   int fw_mapped_iq, int num_descs, unsigned int socket_id)
1521 {
1522         struct lio_gather *g;
1523         int i;
1524
1525         rte_spinlock_init(&lio_dev->glist_lock[iq_no]);
1526
1527         STAILQ_INIT(&lio_dev->glist_head[iq_no]);
1528
1529         for (i = 0; i < num_descs; i++) {
1530                 g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE,
1531                                        socket_id);
1532                 if (g == NULL) {
1533                         lio_dev_err(lio_dev,
1534                                     "lio_gather memory allocation failed for qno %d\n",
1535                                     iq_no);
1536                         break;
1537                 }
1538
1539                 g->sg_size =
1540                     ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE);
1541
1542                 g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8,
1543                                            RTE_CACHE_LINE_SIZE, socket_id);
1544                 if (g->sg == NULL) {
1545                         lio_dev_err(lio_dev,
1546                                     "sg list memory allocation failed for qno %d\n",
1547                                     iq_no);
1548                         rte_free(g);
1549                         break;
1550                 }
1551
1552                 /* The gather component should be aligned on 64-bit boundary */
1553                 if (((unsigned long)g->sg) & 7) {
1554                         g->adjust = 8 - (((unsigned long)g->sg) & 7);
1555                         g->sg =
1556                             (struct lio_sg_entry *)((unsigned long)g->sg +
1557                                                        g->adjust);
1558                 }
1559
1560                 STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list,
1561                                    entries);
1562         }
1563
1564         if (i != num_descs) {
1565                 lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]);
1566                 return -ENOMEM;
1567         }
1568
1569         return 0;
1570 }
1571
1572 void
1573 lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no)
1574 {
1575         lio_delete_instr_queue(lio_dev, iq_no);
1576         rte_free(lio_dev->instr_queue[iq_no]);
1577         lio_dev->instr_queue[iq_no] = NULL;
1578         lio_dev->num_iqs--;
1579 }
1580
1581 static inline uint32_t
1582 lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no)
1583 {
1584         return ((lio_dev->instr_queue[q_no]->max_count - 1) -
1585                 (uint32_t)rte_atomic64_read(
1586                                 &lio_dev->instr_queue[q_no]->instr_pending));
1587 }
1588
1589 static inline int
1590 lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no)
1591 {
1592         return ((uint32_t)rte_atomic64_read(
1593                                 &lio_dev->instr_queue[q_no]->instr_pending) >=
1594                                 (lio_dev->instr_queue[q_no]->max_count - 2));
1595 }
1596
1597 static int
1598 lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no)
1599 {
1600         struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no];
1601         uint32_t count = 10000;
1602
1603         while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) &&
1604                         --count)
1605                 lio_flush_iq(lio_dev, iq);
1606
1607         return count ? 0 : 1;
1608 }
1609
1610 static void
1611 lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr)
1612 {
1613         struct lio_soft_command *sc = sc_ptr;
1614         struct lio_dev_ctrl_cmd *ctrl_cmd;
1615         struct lio_ctrl_pkt *ctrl_pkt;
1616
1617         ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr;
1618         ctrl_cmd = ctrl_pkt->ctrl_cmd;
1619         ctrl_cmd->cond = 1;
1620
1621         lio_free_soft_command(sc);
1622 }
1623
1624 static inline struct lio_soft_command *
1625 lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev,
1626                       struct lio_ctrl_pkt *ctrl_pkt)
1627 {
1628         struct lio_soft_command *sc = NULL;
1629         uint32_t uddsize, datasize;
1630         uint32_t rdatasize;
1631         uint8_t *data;
1632
1633         uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8);
1634
1635         datasize = OCTEON_CMD_SIZE + uddsize;
1636         rdatasize = (ctrl_pkt->wait_time) ? 16 : 0;
1637
1638         sc = lio_alloc_soft_command(lio_dev, datasize,
1639                                     rdatasize, sizeof(struct lio_ctrl_pkt));
1640         if (sc == NULL)
1641                 return NULL;
1642
1643         rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt));
1644
1645         data = (uint8_t *)sc->virtdptr;
1646
1647         rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE);
1648
1649         lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3);
1650
1651         if (uddsize) {
1652                 /* Endian-Swap for UDD should have been done by caller. */
1653                 rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize);
1654         }
1655
1656         sc->iq_no = (uint32_t)ctrl_pkt->iq_no;
1657
1658         lio_prepare_soft_command(lio_dev, sc,
1659                                  LIO_OPCODE, LIO_OPCODE_CMD,
1660                                  0, 0, 0);
1661
1662         sc->callback = lio_ctrl_cmd_callback;
1663         sc->callback_arg = sc;
1664         sc->wait_time = ctrl_pkt->wait_time;
1665
1666         return sc;
1667 }
1668
1669 int
1670 lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt)
1671 {
1672         struct lio_soft_command *sc = NULL;
1673         int retval;
1674
1675         sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt);
1676         if (sc == NULL) {
1677                 lio_dev_err(lio_dev, "soft command allocation failed\n");
1678                 return -1;
1679         }
1680
1681         retval = lio_send_soft_command(lio_dev, sc);
1682         if (retval == LIO_IQ_SEND_FAILED) {
1683                 lio_free_soft_command(sc);
1684                 lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n",
1685                             lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval);
1686                 return -1;
1687         }
1688
1689         return retval;
1690 }
1691
1692 /** Send data packet to the device
1693  *  @param lio_dev - lio device pointer
1694  *  @param ndata   - control structure with queueing, and buffer information
1695  *
1696  *  @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the
1697  *  queue should be stopped, and LIO_IQ_SEND_OK if it sent okay.
1698  */
1699 static inline int
1700 lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata)
1701 {
1702         return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd,
1703                                 ndata->buf, ndata->datasize, ndata->reqtype);
1704 }
1705
1706 uint16_t
1707 lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
1708 {
1709         struct lio_instr_queue *txq = tx_queue;
1710         union lio_cmd_setup cmdsetup;
1711         struct lio_device *lio_dev;
1712         struct lio_iq_stats *stats;
1713         struct lio_data_pkt ndata;
1714         int i, processed = 0;
1715         struct rte_mbuf *m;
1716         uint32_t tag = 0;
1717         int status = 0;
1718         int iq_no;
1719
1720         lio_dev = txq->lio_dev;
1721         iq_no = txq->txpciq.s.q_no;
1722         stats = &lio_dev->instr_queue[iq_no]->stats;
1723
1724         if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) {
1725                 PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n",
1726                            lio_dev->linfo.link.s.link_up);
1727                 goto xmit_failed;
1728         }
1729
1730         lio_dev_cleanup_iq(lio_dev, iq_no);
1731
1732         for (i = 0; i < nb_pkts; i++) {
1733                 uint32_t pkt_len = 0;
1734
1735                 m = pkts[i];
1736
1737                 /* Prepare the attributes for the data to be passed to BASE. */
1738                 memset(&ndata, 0, sizeof(struct lio_data_pkt));
1739
1740                 ndata.buf = m;
1741
1742                 ndata.q_no = iq_no;
1743                 if (lio_iq_is_full(lio_dev, ndata.q_no)) {
1744                         stats->tx_iq_busy++;
1745                         if (lio_dev_cleanup_iq(lio_dev, iq_no)) {
1746                                 PMD_TX_LOG(lio_dev, ERR,
1747                                            "Transmit failed iq:%d full\n",
1748                                            ndata.q_no);
1749                                 break;
1750                         }
1751                 }
1752
1753                 cmdsetup.cmd_setup64 = 0;
1754                 cmdsetup.s.iq_no = iq_no;
1755
1756                 /* check checksum offload flags to form cmd */
1757                 if (m->ol_flags & PKT_TX_IP_CKSUM)
1758                         cmdsetup.s.ip_csum = 1;
1759
1760                 if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM)
1761                         cmdsetup.s.tnl_csum = 1;
1762                 else if ((m->ol_flags & PKT_TX_TCP_CKSUM) ||
1763                                 (m->ol_flags & PKT_TX_UDP_CKSUM))
1764                         cmdsetup.s.transport_csum = 1;
1765
1766                 if (m->nb_segs == 1) {
1767                         pkt_len = rte_pktmbuf_data_len(m);
1768                         cmdsetup.s.u.datasize = pkt_len;
1769                         lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
1770                                             &cmdsetup, tag);
1771                         ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m);
1772                         ndata.reqtype = LIO_REQTYPE_NORESP_NET;
1773                 } else {
1774                         struct lio_buf_free_info *finfo;
1775                         struct lio_gather *g;
1776                         phys_addr_t phyaddr;
1777                         int i, frags;
1778
1779                         finfo = (struct lio_buf_free_info *)rte_malloc(NULL,
1780                                                         sizeof(*finfo), 0);
1781                         if (finfo == NULL) {
1782                                 PMD_TX_LOG(lio_dev, ERR,
1783                                            "free buffer alloc failed\n");
1784                                 goto xmit_failed;
1785                         }
1786
1787                         rte_spinlock_lock(&lio_dev->glist_lock[iq_no]);
1788                         g = (struct lio_gather *)list_delete_first_node(
1789                                                 &lio_dev->glist_head[iq_no]);
1790                         rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]);
1791                         if (g == NULL) {
1792                                 PMD_TX_LOG(lio_dev, ERR,
1793                                            "Transmit scatter gather: glist null!\n");
1794                                 goto xmit_failed;
1795                         }
1796
1797                         cmdsetup.s.gather = 1;
1798                         cmdsetup.s.u.gatherptrs = m->nb_segs;
1799                         lio_prepare_pci_cmd(lio_dev, &ndata.cmd,
1800                                             &cmdsetup, tag);
1801
1802                         memset(g->sg, 0, g->sg_size);
1803                         g->sg[0].ptr[0] = rte_mbuf_data_dma_addr(m);
1804                         lio_add_sg_size(&g->sg[0], m->data_len, 0);
1805                         pkt_len = m->data_len;
1806                         finfo->mbuf = m;
1807
1808                         /* First seg taken care above */
1809                         frags = m->nb_segs - 1;
1810                         i = 1;
1811                         m = m->next;
1812                         while (frags--) {
1813                                 g->sg[(i >> 2)].ptr[(i & 3)] =
1814                                                 rte_mbuf_data_dma_addr(m);
1815                                 lio_add_sg_size(&g->sg[(i >> 2)],
1816                                                 m->data_len, (i & 3));
1817                                 pkt_len += m->data_len;
1818                                 i++;
1819                                 m = m->next;
1820                         }
1821
1822                         phyaddr = rte_mem_virt2phy(g->sg);
1823                         if (phyaddr == RTE_BAD_PHYS_ADDR) {
1824                                 PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n");
1825                                 goto xmit_failed;
1826                         }
1827
1828                         ndata.cmd.cmd3.dptr = phyaddr;
1829                         ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG;
1830
1831                         finfo->g = g;
1832                         finfo->lio_dev = lio_dev;
1833                         finfo->iq_no = (uint64_t)iq_no;
1834                         ndata.buf = finfo;
1835                 }
1836
1837                 ndata.datasize = pkt_len;
1838
1839                 status = lio_send_data_pkt(lio_dev, &ndata);
1840
1841                 if (unlikely(status == LIO_IQ_SEND_FAILED)) {
1842                         PMD_TX_LOG(lio_dev, ERR, "send failed\n");
1843                         break;
1844                 }
1845
1846                 if (unlikely(status == LIO_IQ_SEND_STOP)) {
1847                         PMD_TX_LOG(lio_dev, DEBUG, "iq full\n");
1848                         /* create space as iq is full */
1849                         lio_dev_cleanup_iq(lio_dev, iq_no);
1850                 }
1851
1852                 stats->tx_done++;
1853                 stats->tx_tot_bytes += pkt_len;
1854                 processed++;
1855         }
1856
1857 xmit_failed:
1858         stats->tx_dropped += (nb_pkts - processed);
1859
1860         return processed;
1861 }
1862
1863 void
1864 lio_dev_clear_queues(struct rte_eth_dev *eth_dev)
1865 {
1866         struct lio_instr_queue *txq;
1867         struct lio_droq *rxq;
1868         uint16_t i;
1869
1870         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1871                 txq = eth_dev->data->tx_queues[i];
1872                 if (txq != NULL) {
1873                         lio_dev_tx_queue_release(txq);
1874                         eth_dev->data->tx_queues[i] = NULL;
1875                 }
1876         }
1877
1878         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1879                 rxq = eth_dev->data->rx_queues[i];
1880                 if (rxq != NULL) {
1881                         lio_dev_rx_queue_release(rxq);
1882                         eth_dev->data->rx_queues[i] = NULL;
1883                 }
1884         }
1885 }