drivers/bus/dpaa/base/qbman/qman.c

   1 /*-
   2  * This file is provided under a dual BSD/GPLv2 license. When using or
   3  * redistributing this file, you may do so under either license.
   4  *
   5  *   BSD LICENSE
   6  *
   7  * Copyright 2008-2016 Freescale Semiconductor Inc.
   8  * Copyright 2017 NXP.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions are met:
  12  * * Redistributions of source code must retain the above copyright
  13  * notice, this list of conditions and the following disclaimer.
  14  * * Redistributions in binary form must reproduce the above copyright
  15  * notice, this list of conditions and the following disclaimer in the
  16  * documentation and/or other materials provided with the distribution.
  17  * * Neither the name of the above-listed copyright holders nor the
  18  * names of any contributors may be used to endorse or promote products
  19  * derived from this software without specific prior written permission.
  20  *
  21  *   GPL LICENSE SUMMARY
  22  *
  23  * ALTERNATIVELY, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") as published by the Free Software
  25  * Foundation, either version 2 of that License or (at your option) any
  26  * later version.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  29  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  31  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
  32  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  33  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  34  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  35  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  36  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  37  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGE.
  39  */
  40
  41 #include "qman.h"
  42 #include <rte_branch_prediction.h>
  43
  44 /* Compilation constants */
  45 #define DQRR_MAXFILL    15
  46 #define EQCR_ITHRESH    4       /* if EQCR congests, interrupt threshold */
  47 #define IRQNAME         "QMan portal %d"
  48 #define MAX_IRQNAME     16      /* big enough for "QMan portal %d" */
  49 /* maximum number of DQRR entries to process in qman_poll() */
  50 #define FSL_QMAN_POLL_LIMIT 8
  51
  52 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
  53  * inter-processor locking only. Note, FQLOCK() is always called either under a
  54  * local_irq_save() or from interrupt context - hence there's no need for irq
  55  * protection (and indeed, attempting to nest irq-protection doesn't work, as
  56  * the "irq en/disable" machinery isn't recursive...).
  57  */
  58 #define FQLOCK(fq) \
  59         do { \
  60                 struct qman_fq *__fq478 = (fq); \
  61                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  62                         spin_lock(&__fq478->fqlock); \
  63         } while (0)
  64 #define FQUNLOCK(fq) \
  65         do { \
  66                 struct qman_fq *__fq478 = (fq); \
  67                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  68                         spin_unlock(&__fq478->fqlock); \
  69         } while (0)
  70
  71 static inline void fq_set(struct qman_fq *fq, u32 mask)
  72 {
  73         dpaa_set_bits(mask, &fq->flags);
  74 }
  75
  76 static inline void fq_clear(struct qman_fq *fq, u32 mask)
  77 {
  78         dpaa_clear_bits(mask, &fq->flags);
  79 }
  80
  81 static inline int fq_isset(struct qman_fq *fq, u32 mask)
  82 {
  83         return fq->flags & mask;
  84 }
  85
  86 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
  87 {
  88         return !(fq->flags & mask);
  89 }
  90
  91 struct qman_portal {
  92         struct qm_portal p;
  93         /* PORTAL_BITS_*** - dynamic, strictly internal */
  94         unsigned long bits;
  95         /* interrupt sources processed by portal_isr(), configurable */
  96         unsigned long irq_sources;
  97         u32 use_eqcr_ci_stashing;
  98         u32 slowpoll;   /* only used when interrupts are off */
  99         /* only 1 volatile dequeue at a time */
 100         struct qman_fq *vdqcr_owned;
 101         u32 sdqcr;
 102         int dqrr_disable_ref;
 103         /* A portal-specific handler for DCP ERNs. If this is NULL, the global
 104          * handler is called instead.
 105          */
 106         qman_cb_dc_ern cb_dc_ern;
 107         /* When the cpu-affine portal is activated, this is non-NULL */
 108         const struct qm_portal_config *config;
 109         struct dpa_rbtree retire_table;
 110         char irqname[MAX_IRQNAME];
 111         /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
 112         struct qman_cgrs *cgrs;
 113         /* linked-list of CSCN handlers. */
 114         struct list_head cgr_cbs;
 115         /* list lock */
 116         spinlock_t cgr_lock;
 117         /* track if memory was allocated by the driver */
 118 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 119         /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
 120          * do byte swaps of DQRR read only memory.  First entry must be aligned
 121          * to 2 ** 10 to ensure DQRR index calculations based shadow copy
 122          * address (6 bits for address shift + 4 bits for the DQRR size).
 123          */
 124         struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
 125                     __attribute__((aligned(1024)));
 126 #endif
 127 };
 128
 129 /* Global handler for DCP ERNs. Used when the portal receiving the message does
 130  * not have a portal-specific handler.
 131  */
 132 static qman_cb_dc_ern cb_dc_ern;
 133
 134 static cpumask_t affine_mask;
 135 static DEFINE_SPINLOCK(affine_mask_lock);
 136 static u16 affine_channels[NR_CPUS];
 137 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
 138
 139 static inline struct qman_portal *get_affine_portal(void)
 140 {
 141         return &RTE_PER_LCORE(qman_affine_portal);
 142 }
 143
 144 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
 145  * retirement notifications (the fact they are sometimes h/w-consumed means that
 146  * contextB isn't always a s/w demux - and as we can't know which case it is
 147  * when looking at the notification, we have to use the slow lookup for all of
 148  * them). NB, it's possible to have multiple FQ objects refer to the same FQID
 149  * (though at most one of them should be the consumer), so this table isn't for
 150  * all FQs - FQs are added when retirement commands are issued, and removed when
 151  * they complete, which also massively reduces the size of this table.
 152  */
 153 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
 154 /*
 155  * This is what everything can wait on, even if it migrates to a different cpu
 156  * to the one whose affine portal it is waiting on.
 157  */
 158 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
 159
 160 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
 161 {
 162         int ret = fqtree_push(&p->retire_table, fq);
 163
 164         if (ret)
 165                 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
 166         return ret;
 167 }
 168
 169 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
 170 {
 171         fqtree_del(&p->retire_table, fq);
 172 }
 173
 174 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
 175 {
 176         return fqtree_find(&p->retire_table, fqid);
 177 }
 178
 179 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 180 static void **qman_fq_lookup_table;
 181 static size_t qman_fq_lookup_table_size;
 182
 183 int qman_setup_fq_lookup_table(size_t num_entries)
 184 {
 185         num_entries++;
 186         /* Allocate 1 more entry since the first entry is not used */
 187         qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
 188         if (!qman_fq_lookup_table) {
 189                 pr_err("QMan: Could not allocate fq lookup table\n");
 190                 return -ENOMEM;
 191         }
 192         memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
 193         qman_fq_lookup_table_size = num_entries;
 194         pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
 195                 qman_fq_lookup_table,
 196                         (unsigned long)qman_fq_lookup_table_size);
 197         return 0;
 198 }
 199
 200 /* global structure that maintains fq object mapping */
 201 static DEFINE_SPINLOCK(fq_hash_table_lock);
 202
 203 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
 204 {
 205         u32 i;
 206
 207         spin_lock(&fq_hash_table_lock);
 208         /* Can't use index zero because this has special meaning
 209          * in context_b field.
 210          */
 211         for (i = 1; i < qman_fq_lookup_table_size; i++) {
 212                 if (qman_fq_lookup_table[i] == NULL) {
 213                         *entry = i;
 214                         qman_fq_lookup_table[i] = fq;
 215                         spin_unlock(&fq_hash_table_lock);
 216                         return 0;
 217                 }
 218         }
 219         spin_unlock(&fq_hash_table_lock);
 220         return -ENOMEM;
 221 }
 222
 223 static void clear_fq_table_entry(u32 entry)
 224 {
 225         spin_lock(&fq_hash_table_lock);
 226         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 227         qman_fq_lookup_table[entry] = NULL;
 228         spin_unlock(&fq_hash_table_lock);
 229 }
 230
 231 static inline struct qman_fq *get_fq_table_entry(u32 entry)
 232 {
 233         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 234         return qman_fq_lookup_table[entry];
 235 }
 236 #endif
 237
 238 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
 239 {
 240         /* Byteswap the FQD to HW format */
 241         fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
 242         fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
 243         fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
 244         fqd->context_b = cpu_to_be32(fqd->context_b);
 245         fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
 246         fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
 247 }
 248
 249 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
 250 {
 251         /* Byteswap the FQD to CPU format */
 252         fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
 253         fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
 254         fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
 255         fqd->context_b = be32_to_cpu(fqd->context_b);
 256         fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
 257 }
 258
 259 static inline void cpu_to_hw_fd(struct qm_fd *fd)
 260 {
 261         fd->addr = cpu_to_be40(fd->addr);
 262         fd->status = cpu_to_be32(fd->status);
 263         fd->opaque = cpu_to_be32(fd->opaque);
 264 }
 265
 266 static inline void hw_fd_to_cpu(struct qm_fd *fd)
 267 {
 268         fd->addr = be40_to_cpu(fd->addr);
 269         fd->status = be32_to_cpu(fd->status);
 270         fd->opaque = be32_to_cpu(fd->opaque);
 271 }
 272
 273 /* In the case that slow- and fast-path handling are both done by qman_poll()
 274  * (ie. because there is no interrupt handling), we ought to balance how often
 275  * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
 276  * sources, so we call the fast poll 'n' times before calling the slow poll
 277  * once. The idle decrementer constant is used when the last slow-poll detected
 278  * no work to do, and the busy decrementer constant when the last slow-poll had
 279  * work to do.
 280  */
 281 #define SLOW_POLL_IDLE   1000
 282 #define SLOW_POLL_BUSY   10
 283 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
 284 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 285                                               unsigned int poll_limit);
 286
 287 /* Portal interrupt handler */
 288 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
 289 {
 290         struct qman_portal *p = ptr;
 291         /*
 292          * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
 293          * it could race against a Query Congestion State command also given
 294          * as part of the handling of this interrupt source. We mustn't
 295          * clear it a second time in this top-level function.
 296          */
 297         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
 298                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
 299         u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
 300         /* DQRR-handling if it's interrupt-driven */
 301         if (is & QM_PIRQ_DQRI)
 302                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
 303         /* Handling of anything else that's interrupt-driven */
 304         clear |= __poll_portal_slow(p, is);
 305         qm_isr_status_clear(&p->p, clear);
 306         return IRQ_HANDLED;
 307 }
 308
 309 /* This inner version is used privately by qman_create_affine_portal(), as well
 310  * as by the exported qman_stop_dequeues().
 311  */
 312 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
 313 {
 314         if (!(p->dqrr_disable_ref++))
 315                 qm_dqrr_set_maxfill(&p->p, 0);
 316 }
 317
 318 static int drain_mr_fqrni(struct qm_portal *p)
 319 {
 320         const struct qm_mr_entry *msg;
 321 loop:
 322         msg = qm_mr_current(p);
 323         if (!msg) {
 324                 /*
 325                  * if MR was full and h/w had other FQRNI entries to produce, we
 326                  * need to allow it time to produce those entries once the
 327                  * existing entries are consumed. A worst-case situation
 328                  * (fully-loaded system) means h/w sequencers may have to do 3-4
 329                  * other things before servicing the portal's MR pump, each of
 330                  * which (if slow) may take ~50 qman cycles (which is ~200
 331                  * processor cycles). So rounding up and then multiplying this
 332                  * worst-case estimate by a factor of 10, just to be
 333                  * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
 334                  * one entry at a time, so h/w has an opportunity to produce new
 335                  * entries well before the ring has been fully consumed, so
 336                  * we're being *really* paranoid here.
 337                  */
 338                 u64 now, then = mfatb();
 339
 340                 do {
 341                         now = mfatb();
 342                 } while ((then + 10000) > now);
 343                 msg = qm_mr_current(p);
 344                 if (!msg)
 345                         return 0;
 346         }
 347         if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
 348                 /* We aren't draining anything but FQRNIs */
 349                 pr_err("Found verb 0x%x in MR\n", msg->verb);
 350                 return -1;
 351         }
 352         qm_mr_next(p);
 353         qm_mr_cci_consume(p, 1);
 354         goto loop;
 355 }
 356
 357 static inline int qm_eqcr_init(struct qm_portal *portal,
 358                                enum qm_eqcr_pmode pmode,
 359                                unsigned int eq_stash_thresh,
 360                                int eq_stash_prio)
 361 {
 362         /* This use of 'register', as well as all other occurrences, is because
 363          * it has been observed to generate much faster code with gcc than is
 364          * otherwise the case.
 365          */
 366         register struct qm_eqcr *eqcr = &portal->eqcr;
 367         u32 cfg;
 368         u8 pi;
 369
 370         eqcr->ring = portal->addr.ce + QM_CL_EQCR;
 371         eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 372         qm_cl_invalidate(EQCR_CI);
 373         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 374         eqcr->cursor = eqcr->ring + pi;
 375         eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
 376                         QM_EQCR_VERB_VBIT : 0;
 377         eqcr->available = QM_EQCR_SIZE - 1 -
 378                         qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
 379         eqcr->ithresh = qm_in(EQCR_ITR);
 380 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 381         eqcr->busy = 0;
 382         eqcr->pmode = pmode;
 383 #endif
 384         cfg = (qm_in(CFG) & 0x00ffffff) |
 385                 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
 386                 (eq_stash_prio << 26)   | /* QCSP_CFG: EP */
 387                 ((pmode & 0x3) << 24);  /* QCSP_CFG::EPM */
 388         qm_out(CFG, cfg);
 389         return 0;
 390 }
 391
 392 static inline void qm_eqcr_finish(struct qm_portal *portal)
 393 {
 394         register struct qm_eqcr *eqcr = &portal->eqcr;
 395         u8 pi, ci;
 396         u32 cfg;
 397
 398         /*
 399          * Disable EQCI stashing because the QMan only
 400          * presents the value it previously stashed to
 401          * maintain coherency.  Setting the stash threshold
 402          * to 1 then 0 ensures that QMan has resyncronized
 403          * its internal copy so that the portal is clean
 404          * when it is reinitialized in the future
 405          */
 406         cfg = (qm_in(CFG) & 0x0fffffff) |
 407                 (1 << 28); /* QCSP_CFG: EST */
 408         qm_out(CFG, cfg);
 409         cfg &= 0x0fffffff; /* stash threshold = 0 */
 410         qm_out(CFG, cfg);
 411
 412         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 413         ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 414
 415         /* Refresh EQCR CI cache value */
 416         qm_cl_invalidate(EQCR_CI);
 417         eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
 418
 419 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 420         DPAA_ASSERT(!eqcr->busy);
 421 #endif
 422         if (pi != EQCR_PTR2IDX(eqcr->cursor))
 423                 pr_crit("losing uncommitted EQCR entries\n");
 424         if (ci != eqcr->ci)
 425                 pr_crit("missing existing EQCR completions\n");
 426         if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
 427                 pr_crit("EQCR destroyed unquiesced\n");
 428 }
 429
 430 static inline int qm_dqrr_init(struct qm_portal *portal,
 431                         __maybe_unused const struct qm_portal_config *config,
 432                         enum qm_dqrr_dmode dmode,
 433                         __maybe_unused enum qm_dqrr_pmode pmode,
 434                         enum qm_dqrr_cmode cmode, u8 max_fill)
 435 {
 436         register struct qm_dqrr *dqrr = &portal->dqrr;
 437         u32 cfg;
 438
 439         /* Make sure the DQRR will be idle when we enable */
 440         qm_out(DQRR_SDQCR, 0);
 441         qm_out(DQRR_VDQCR, 0);
 442         qm_out(DQRR_PDQCR, 0);
 443         dqrr->ring = portal->addr.ce + QM_CL_DQRR;
 444         dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
 445         dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
 446         dqrr->cursor = dqrr->ring + dqrr->ci;
 447         dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
 448         dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
 449                         QM_DQRR_VERB_VBIT : 0;
 450         dqrr->ithresh = qm_in(DQRR_ITR);
 451 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 452         dqrr->dmode = dmode;
 453         dqrr->pmode = pmode;
 454         dqrr->cmode = cmode;
 455 #endif
 456         /* Invalidate every ring entry before beginning */
 457         for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
 458                 dccivac(qm_cl(dqrr->ring, cfg));
 459         cfg = (qm_in(CFG) & 0xff000f00) |
 460                 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
 461                 ((dmode & 1) << 18) |                   /* DP */
 462                 ((cmode & 3) << 16) |                   /* DCM */
 463                 0xa0 |                                  /* RE+SE */
 464                 (0 ? 0x40 : 0) |                        /* Ignore RP */
 465                 (0 ? 0x10 : 0);                         /* Ignore SP */
 466         qm_out(CFG, cfg);
 467         qm_dqrr_set_maxfill(portal, max_fill);
 468         return 0;
 469 }
 470
 471 static inline void qm_dqrr_finish(struct qm_portal *portal)
 472 {
 473         __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
 474 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 475         if ((dqrr->cmode != qm_dqrr_cdc) &&
 476             (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
 477                 pr_crit("Ignoring completed DQRR entries\n");
 478 #endif
 479 }
 480
 481 static inline int qm_mr_init(struct qm_portal *portal,
 482                              __maybe_unused enum qm_mr_pmode pmode,
 483                              enum qm_mr_cmode cmode)
 484 {
 485         register struct qm_mr *mr = &portal->mr;
 486         u32 cfg;
 487
 488         mr->ring = portal->addr.ce + QM_CL_MR;
 489         mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
 490         mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
 491         mr->cursor = mr->ring + mr->ci;
 492         mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
 493         mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
 494         mr->ithresh = qm_in(MR_ITR);
 495 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 496         mr->pmode = pmode;
 497         mr->cmode = cmode;
 498 #endif
 499         cfg = (qm_in(CFG) & 0xfffff0ff) |
 500                 ((cmode & 1) << 8);             /* QCSP_CFG:MM */
 501         qm_out(CFG, cfg);
 502         return 0;
 503 }
 504
 505 static inline void qm_mr_pvb_update(struct qm_portal *portal)
 506 {
 507         register struct qm_mr *mr = &portal->mr;
 508         const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
 509
 510 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 511         DPAA_ASSERT(mr->pmode == qm_mr_pvb);
 512 #endif
 513         /* when accessing 'verb', use __raw_readb() to ensure that compiler
 514          * inlining doesn't try to optimise out "excess reads".
 515          */
 516         if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
 517                 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
 518                 if (!mr->pi)
 519                         mr->vbit ^= QM_MR_VERB_VBIT;
 520                 mr->fill++;
 521                 res = MR_INC(res);
 522         }
 523         dcbit_ro(res);
 524 }
 525
 526 static inline
 527 struct qman_portal *qman_create_portal(
 528                         struct qman_portal *portal,
 529                               const struct qm_portal_config *c,
 530                               const struct qman_cgrs *cgrs)
 531 {
 532         struct qm_portal *p;
 533         char buf[16];
 534         int ret;
 535         u32 isdr;
 536
 537         p = &portal->p;
 538
 539         portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
 540         /*
 541          * prep the low-level portal struct with the mapped addresses from the
 542          * config, everything that follows depends on it and "config" is more
 543          * for (de)reference
 544          */
 545         p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
 546         p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
 547         /*
 548          * If CI-stashing is used, the current defaults use a threshold of 3,
 549          * and stash with high-than-DQRR priority.
 550          */
 551         if (qm_eqcr_init(p, qm_eqcr_pvb,
 552                          portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
 553                 pr_err("Qman EQCR initialisation failed\n");
 554                 goto fail_eqcr;
 555         }
 556         if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
 557                          qm_dqrr_cdc, DQRR_MAXFILL)) {
 558                 pr_err("Qman DQRR initialisation failed\n");
 559                 goto fail_dqrr;
 560         }
 561         if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
 562                 pr_err("Qman MR initialisation failed\n");
 563                 goto fail_mr;
 564         }
 565         if (qm_mc_init(p)) {
 566                 pr_err("Qman MC initialisation failed\n");
 567                 goto fail_mc;
 568         }
 569
 570         /* static interrupt-gating controls */
 571         qm_dqrr_set_ithresh(p, 0);
 572         qm_mr_set_ithresh(p, 0);
 573         qm_isr_set_iperiod(p, 0);
 574         portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
 575         if (!portal->cgrs)
 576                 goto fail_cgrs;
 577         /* initial snapshot is no-depletion */
 578         qman_cgrs_init(&portal->cgrs[1]);
 579         if (cgrs)
 580                 portal->cgrs[0] = *cgrs;
 581         else
 582                 /* if the given mask is NULL, assume all CGRs can be seen */
 583                 qman_cgrs_fill(&portal->cgrs[0]);
 584         INIT_LIST_HEAD(&portal->cgr_cbs);
 585         spin_lock_init(&portal->cgr_lock);
 586         portal->bits = 0;
 587         portal->slowpoll = 0;
 588         portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
 589                         QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
 590                         QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
 591         portal->dqrr_disable_ref = 0;
 592         portal->cb_dc_ern = NULL;
 593         sprintf(buf, "qportal-%d", c->channel);
 594         dpa_rbtree_init(&portal->retire_table);
 595         isdr = 0xffffffff;
 596         qm_isr_disable_write(p, isdr);
 597         portal->irq_sources = 0;
 598         qm_isr_enable_write(p, portal->irq_sources);
 599         qm_isr_status_clear(p, 0xffffffff);
 600         snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 601         if (request_irq(c->irq, portal_isr, 0, portal->irqname,
 602                         portal)) {
 603                 pr_err("request_irq() failed\n");
 604                 goto fail_irq;
 605         }
 606
 607         /* Need EQCR to be empty before continuing */
 608         isdr &= ~QM_PIRQ_EQCI;
 609         qm_isr_disable_write(p, isdr);
 610         ret = qm_eqcr_get_fill(p);
 611         if (ret) {
 612                 pr_err("Qman EQCR unclean\n");
 613                 goto fail_eqcr_empty;
 614         }
 615         isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
 616         qm_isr_disable_write(p, isdr);
 617         if (qm_dqrr_current(p)) {
 618                 pr_err("Qman DQRR unclean\n");
 619                 qm_dqrr_cdc_consume_n(p, 0xffff);
 620         }
 621         if (qm_mr_current(p) && drain_mr_fqrni(p)) {
 622                 /* special handling, drain just in case it's a few FQRNIs */
 623                 if (drain_mr_fqrni(p))
 624                         goto fail_dqrr_mr_empty;
 625         }
 626         /* Success */
 627         portal->config = c;
 628         qm_isr_disable_write(p, 0);
 629         qm_isr_uninhibit(p);
 630         /* Write a sane SDQCR */
 631         qm_dqrr_sdqcr_set(p, portal->sdqcr);
 632         return portal;
 633 fail_dqrr_mr_empty:
 634 fail_eqcr_empty:
 635         free_irq(c->irq, portal);
 636 fail_irq:
 637         kfree(portal->cgrs);
 638         spin_lock_destroy(&portal->cgr_lock);
 639 fail_cgrs:
 640         qm_mc_finish(p);
 641 fail_mc:
 642         qm_mr_finish(p);
 643 fail_mr:
 644         qm_dqrr_finish(p);
 645 fail_dqrr:
 646         qm_eqcr_finish(p);
 647 fail_eqcr:
 648         return NULL;
 649 }
 650
 651 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
 652                                               const struct qman_cgrs *cgrs)
 653 {
 654         struct qman_portal *res;
 655         struct qman_portal *portal = get_affine_portal();
 656         /* A criteria for calling this function (from qman_driver.c) is that
 657          * we're already affine to the cpu and won't schedule onto another cpu.
 658          */
 659
 660         res = qman_create_portal(portal, c, cgrs);
 661         if (res) {
 662                 spin_lock(&affine_mask_lock);
 663                 CPU_SET(c->cpu, &affine_mask);
 664                 affine_channels[c->cpu] =
 665                         c->channel;
 666                 spin_unlock(&affine_mask_lock);
 667         }
 668         return res;
 669 }
 670
 671 static inline
 672 void qman_destroy_portal(struct qman_portal *qm)
 673 {
 674         const struct qm_portal_config *pcfg;
 675
 676         /* Stop dequeues on the portal */
 677         qm_dqrr_sdqcr_set(&qm->p, 0);
 678
 679         /*
 680          * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
 681          * something related to QM_PIRQ_EQCI, this may need fixing.
 682          * Also, due to the prefetching model used for CI updates in the enqueue
 683          * path, this update will only invalidate the CI cacheline *after*
 684          * working on it, so we need to call this twice to ensure a full update
 685          * irrespective of where the enqueue processing was at when the teardown
 686          * began.
 687          */
 688         qm_eqcr_cce_update(&qm->p);
 689         qm_eqcr_cce_update(&qm->p);
 690         pcfg = qm->config;
 691
 692         free_irq(pcfg->irq, qm);
 693
 694         kfree(qm->cgrs);
 695         qm_mc_finish(&qm->p);
 696         qm_mr_finish(&qm->p);
 697         qm_dqrr_finish(&qm->p);
 698         qm_eqcr_finish(&qm->p);
 699
 700         qm->config = NULL;
 701
 702         spin_lock_destroy(&qm->cgr_lock);
 703 }
 704
 705 const struct qm_portal_config *qman_destroy_affine_portal(void)
 706 {
 707         /* We don't want to redirect if we're a slave, use "raw" */
 708         struct qman_portal *qm = get_affine_portal();
 709         const struct qm_portal_config *pcfg;
 710         int cpu;
 711
 712         pcfg = qm->config;
 713         cpu = pcfg->cpu;
 714
 715         qman_destroy_portal(qm);
 716
 717         spin_lock(&affine_mask_lock);
 718         CPU_CLR(cpu, &affine_mask);
 719         spin_unlock(&affine_mask_lock);
 720         return pcfg;
 721 }
 722
 723 int qman_get_portal_index(void)
 724 {
 725         struct qman_portal *p = get_affine_portal();
 726         return p->config->index;
 727 }
 728
 729 /* Inline helper to reduce nesting in __poll_portal_slow() */
 730 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
 731                                    const struct qm_mr_entry *msg, u8 verb)
 732 {
 733         FQLOCK(fq);
 734         switch (verb) {
 735         case QM_MR_VERB_FQRL:
 736                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
 737                 fq_clear(fq, QMAN_FQ_STATE_ORL);
 738                 table_del_fq(p, fq);
 739                 break;
 740         case QM_MR_VERB_FQRN:
 741                 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
 742                             (fq->state == qman_fq_state_sched));
 743                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
 744                 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
 745                 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
 746                         fq_set(fq, QMAN_FQ_STATE_NE);
 747                 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
 748                         fq_set(fq, QMAN_FQ_STATE_ORL);
 749                 else
 750                         table_del_fq(p, fq);
 751                 fq->state = qman_fq_state_retired;
 752                 break;
 753         case QM_MR_VERB_FQPN:
 754                 DPAA_ASSERT(fq->state == qman_fq_state_sched);
 755                 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
 756                 fq->state = qman_fq_state_parked;
 757         }
 758         FQUNLOCK(fq);
 759 }
 760
 761 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
 762 {
 763         const struct qm_mr_entry *msg;
 764         struct qm_mr_entry swapped_msg;
 765
 766         if (is & QM_PIRQ_CSCI) {
 767                 struct qman_cgrs rr, c;
 768                 struct qm_mc_result *mcr;
 769                 struct qman_cgr *cgr;
 770
 771                 spin_lock(&p->cgr_lock);
 772                 /*
 773                  * The CSCI bit must be cleared _before_ issuing the
 774                  * Query Congestion State command, to ensure that a long
 775                  * CGR State Change callback cannot miss an intervening
 776                  * state change.
 777                  */
 778                 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
 779                 qm_mc_start(&p->p);
 780                 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
 781                 while (!(mcr = qm_mc_result(&p->p)))
 782                         cpu_relax();
 783                 /* mask out the ones I'm not interested in */
 784                 qman_cgrs_and(&rr, (const struct qman_cgrs *)
 785                         &mcr->querycongestion.state, &p->cgrs[0]);
 786                 /* check previous snapshot for delta, enter/exit congestion */
 787                 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
 788                 /* update snapshot */
 789                 qman_cgrs_cp(&p->cgrs[1], &rr);
 790                 /* Invoke callback */
 791                 list_for_each_entry(cgr, &p->cgr_cbs, node)
 792                         if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
 793                                 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
 794                 spin_unlock(&p->cgr_lock);
 795         }
 796
 797         if (is & QM_PIRQ_EQRI) {
 798                 qm_eqcr_cce_update(&p->p);
 799                 qm_eqcr_set_ithresh(&p->p, 0);
 800                 wake_up(&affine_queue);
 801         }
 802
 803         if (is & QM_PIRQ_MRI) {
 804                 struct qman_fq *fq;
 805                 u8 verb, num = 0;
 806 mr_loop:
 807                 qm_mr_pvb_update(&p->p);
 808                 msg = qm_mr_current(&p->p);
 809                 if (!msg)
 810                         goto mr_done;
 811                 swapped_msg = *msg;
 812                 hw_fd_to_cpu(&swapped_msg.ern.fd);
 813                 verb = msg->verb & QM_MR_VERB_TYPE_MASK;
 814                 /* The message is a software ERN iff the 0x20 bit is set */
 815                 if (verb & 0x20) {
 816                         switch (verb) {
 817                         case QM_MR_VERB_FQRNI:
 818                                 /* nada, we drop FQRNIs on the floor */
 819                                 break;
 820                         case QM_MR_VERB_FQRN:
 821                         case QM_MR_VERB_FQRL:
 822                                 /* Lookup in the retirement table */
 823                                 fq = table_find_fq(p,
 824                                                    be32_to_cpu(msg->fq.fqid));
 825                                 DPAA_BUG_ON(!fq);
 826                                 fq_state_change(p, fq, &swapped_msg, verb);
 827                                 if (fq->cb.fqs)
 828                                         fq->cb.fqs(p, fq, &swapped_msg);
 829                                 break;
 830                         case QM_MR_VERB_FQPN:
 831                                 /* Parked */
 832 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 833                                 fq = get_fq_table_entry(
 834                                         be32_to_cpu(msg->fq.contextB));
 835 #else
 836                                 fq = (void *)(uintptr_t)
 837                                         be32_to_cpu(msg->fq.contextB);
 838 #endif
 839                                 fq_state_change(p, fq, msg, verb);
 840                                 if (fq->cb.fqs)
 841                                         fq->cb.fqs(p, fq, &swapped_msg);
 842                                 break;
 843                         case QM_MR_VERB_DC_ERN:
 844                                 /* DCP ERN */
 845                                 if (p->cb_dc_ern)
 846                                         p->cb_dc_ern(p, msg);
 847                                 else if (cb_dc_ern)
 848                                         cb_dc_ern(p, msg);
 849                                 else {
 850                                         static int warn_once;
 851
 852                                         if (!warn_once) {
 853                                                 pr_crit("Leaking DCP ERNs!\n");
 854                                                 warn_once = 1;
 855                                         }
 856                                 }
 857                                 break;
 858                         default:
 859                                 pr_crit("Invalid MR verb 0x%02x\n", verb);
 860                         }
 861                 } else {
 862                         /* Its a software ERN */
 863 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 864                         fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
 865 #else
 866                         fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
 867 #endif
 868                         fq->cb.ern(p, fq, &swapped_msg);
 869                 }
 870                 num++;
 871                 qm_mr_next(&p->p);
 872                 goto mr_loop;
 873 mr_done:
 874                 qm_mr_cci_consume(&p->p, num);
 875         }
 876         /*
 877          * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
 878          * processing. If that interrupt source has meanwhile been re-asserted,
 879          * we mustn't clear it here (or in the top-level interrupt handler).
 880          */
 881         return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
 882 }
 883
 884 /*
 885  * remove some slowish-path stuff from the "fast path" and make sure it isn't
 886  * inlined.
 887  */
 888 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
 889 {
 890         p->vdqcr_owned = NULL;
 891         FQLOCK(fq);
 892         fq_clear(fq, QMAN_FQ_STATE_VDQCR);
 893         FQUNLOCK(fq);
 894         wake_up(&affine_queue);
 895 }
 896
 897 /*
 898  * The only states that would conflict with other things if they ran at the
 899  * same time on the same cpu are:
 900  *
 901  *   (i) setting/clearing vdqcr_owned, and
 902  *  (ii) clearing the NE (Not Empty) flag.
 903  *
 904  * Both are safe. Because;
 905  *
 906  *   (i) this clearing can only occur after qman_set_vdq() has set the
 907  *       vdqcr_owned field (which it does before setting VDQCR), and
 908  *       qman_volatile_dequeue() blocks interrupts and preemption while this is
 909  *       done so that we can't interfere.
 910  *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
 911  *       with (i) that API prevents us from interfering until it's safe.
 912  *
 913  * The good thing is that qman_set_vdq() and qman_retire_fq() run far
 914  * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
 915  * advantage comes from this function not having to "lock" anything at all.
 916  *
 917  * Note also that the callbacks are invoked at points which are safe against the
 918  * above potential conflicts, but that this function itself is not re-entrant
 919  * (this is because the function tracks one end of each FIFO in the portal and
 920  * we do *not* want to lock that). So the consequence is that it is safe for
 921  * user callbacks to call into any QMan API.
 922  */
 923 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 924                                               unsigned int poll_limit)
 925 {
 926         const struct qm_dqrr_entry *dq;
 927         struct qman_fq *fq;
 928         enum qman_cb_dqrr_result res;
 929         unsigned int limit = 0;
 930 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 931         struct qm_dqrr_entry *shadow;
 932 #endif
 933         do {
 934                 qm_dqrr_pvb_update(&p->p);
 935                 dq = qm_dqrr_current(&p->p);
 936                 if (!dq)
 937                         break;
 938 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 939         /* If running on an LE system the fields of the
 940          * dequeue entry must be swapper.  Because the
 941          * QMan HW will ignore writes the DQRR entry is
 942          * copied and the index stored within the copy
 943          */
 944                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
 945                 *shadow = *dq;
 946                 dq = shadow;
 947                 shadow->fqid = be32_to_cpu(shadow->fqid);
 948                 shadow->contextB = be32_to_cpu(shadow->contextB);
 949                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
 950                 hw_fd_to_cpu(&shadow->fd);
 951 #endif
 952
 953                 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
 954                         /*
 955                          * VDQCR: don't trust context_b as the FQ may have
 956                          * been configured for h/w consumption and we're
 957                          * draining it post-retirement.
 958                          */
 959                         fq = p->vdqcr_owned;
 960                         /*
 961                          * We only set QMAN_FQ_STATE_NE when retiring, so we
 962                          * only need to check for clearing it when doing
 963                          * volatile dequeues.  It's one less thing to check
 964                          * in the critical path (SDQCR).
 965                          */
 966                         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
 967                                 fq_clear(fq, QMAN_FQ_STATE_NE);
 968                         /*
 969                          * This is duplicated from the SDQCR code, but we
 970                          * have stuff to do before *and* after this callback,
 971                          * and we don't want multiple if()s in the critical
 972                          * path (SDQCR).
 973                          */
 974                         res = fq->cb.dqrr(p, fq, dq);
 975                         if (res == qman_cb_dqrr_stop)
 976                                 break;
 977                         /* Check for VDQCR completion */
 978                         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
 979                                 clear_vdqcr(p, fq);
 980                 } else {
 981                         /* SDQCR: context_b points to the FQ */
 982 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 983                         fq = get_fq_table_entry(dq->contextB);
 984 #else
 985                         fq = (void *)(uintptr_t)dq->contextB;
 986 #endif
 987                         /* Now let the callback do its stuff */
 988                         res = fq->cb.dqrr(p, fq, dq);
 989                         /*
 990                          * The callback can request that we exit without
 991                          * consuming this entry nor advancing;
 992                          */
 993                         if (res == qman_cb_dqrr_stop)
 994                                 break;
 995                 }
 996                 /* Interpret 'dq' from a driver perspective. */
 997                 /*
 998                  * Parking isn't possible unless HELDACTIVE was set. NB,
 999                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1000                  * check for HELDACTIVE to cover both.
1001                  */
1002                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1003                             (res != qman_cb_dqrr_park));
1004                 /* just means "skip it, I'll consume it myself later on" */
1005                 if (res != qman_cb_dqrr_defer)
1006                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1007                                                  res == qman_cb_dqrr_park);
1008                 /* Move forward */
1009                 qm_dqrr_next(&p->p);
1010                 /*
1011                  * Entry processed and consumed, increment our counter.  The
1012                  * callback can request that we exit after consuming the
1013                  * entry, and we also exit if we reach our processing limit,
1014                  * so loop back only if neither of these conditions is met.
1015                  */
1016         } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1017
1018         return limit;
1019 }
1020
1021 u16 qman_affine_channel(int cpu)
1022 {
1023         if (cpu < 0) {
1024                 struct qman_portal *portal = get_affine_portal();
1025
1026                 cpu = portal->config->cpu;
1027         }
1028         DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1029         return affine_channels[cpu];
1030 }
1031
1032 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1033 {
1034         struct qman_portal *p = get_affine_portal();
1035         const struct qm_dqrr_entry *dq;
1036 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1037         struct qm_dqrr_entry *shadow;
1038 #endif
1039
1040         qm_dqrr_pvb_update(&p->p);
1041         dq = qm_dqrr_current(&p->p);
1042         if (!dq)
1043                 return NULL;
1044
1045         if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1046                 /* Invalid DQRR - put the portal and consume the DQRR.
1047                  * Return NULL to user as no packet is seen.
1048                  */
1049                 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1050                 return NULL;
1051         }
1052
1053 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1054         shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1055         *shadow = *dq;
1056         dq = shadow;
1057         shadow->fqid = be32_to_cpu(shadow->fqid);
1058         shadow->contextB = be32_to_cpu(shadow->contextB);
1059         shadow->seqnum = be16_to_cpu(shadow->seqnum);
1060         hw_fd_to_cpu(&shadow->fd);
1061 #endif
1062
1063         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1064                 fq_clear(fq, QMAN_FQ_STATE_NE);
1065
1066         return (struct qm_dqrr_entry *)dq;
1067 }
1068
1069 void qman_dqrr_consume(struct qman_fq *fq,
1070                        struct qm_dqrr_entry *dq)
1071 {
1072         struct qman_portal *p = get_affine_portal();
1073
1074         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1075                 clear_vdqcr(p, fq);
1076
1077         qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1078         qm_dqrr_next(&p->p);
1079 }
1080
1081 int qman_poll_dqrr(unsigned int limit)
1082 {
1083         struct qman_portal *p = get_affine_portal();
1084         int ret;
1085
1086         ret = __poll_portal_fast(p, limit);
1087         return ret;
1088 }
1089
1090 void qman_poll(void)
1091 {
1092         struct qman_portal *p = get_affine_portal();
1093
1094         if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1095                 if (!(p->slowpoll--)) {
1096                         u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1097                         u32 active = __poll_portal_slow(p, is);
1098
1099                         if (active) {
1100                                 qm_isr_status_clear(&p->p, active);
1101                                 p->slowpoll = SLOW_POLL_BUSY;
1102                         } else
1103                                 p->slowpoll = SLOW_POLL_IDLE;
1104                 }
1105         }
1106         if ((~p->irq_sources) & QM_PIRQ_DQRI)
1107                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1108 }
1109
1110 void qman_stop_dequeues(void)
1111 {
1112         struct qman_portal *p = get_affine_portal();
1113
1114         qman_stop_dequeues_ex(p);
1115 }
1116
1117 void qman_start_dequeues(void)
1118 {
1119         struct qman_portal *p = get_affine_portal();
1120
1121         DPAA_ASSERT(p->dqrr_disable_ref > 0);
1122         if (!(--p->dqrr_disable_ref))
1123                 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1124 }
1125
1126 void qman_static_dequeue_add(u32 pools)
1127 {
1128         struct qman_portal *p = get_affine_portal();
1129
1130         pools &= p->config->pools;
1131         p->sdqcr |= pools;
1132         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1133 }
1134
1135 void qman_static_dequeue_del(u32 pools)
1136 {
1137         struct qman_portal *p = get_affine_portal();
1138
1139         pools &= p->config->pools;
1140         p->sdqcr &= ~pools;
1141         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1142 }
1143
1144 u32 qman_static_dequeue_get(void)
1145 {
1146         struct qman_portal *p = get_affine_portal();
1147         return p->sdqcr;
1148 }
1149
1150 void qman_dca(struct qm_dqrr_entry *dq, int park_request)
1151 {
1152         struct qman_portal *p = get_affine_portal();
1153
1154         qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1155 }
1156
1157 /* Frame queue API */
1158 static const char *mcr_result_str(u8 result)
1159 {
1160         switch (result) {
1161         case QM_MCR_RESULT_NULL:
1162                 return "QM_MCR_RESULT_NULL";
1163         case QM_MCR_RESULT_OK:
1164                 return "QM_MCR_RESULT_OK";
1165         case QM_MCR_RESULT_ERR_FQID:
1166                 return "QM_MCR_RESULT_ERR_FQID";
1167         case QM_MCR_RESULT_ERR_FQSTATE:
1168                 return "QM_MCR_RESULT_ERR_FQSTATE";
1169         case QM_MCR_RESULT_ERR_NOTEMPTY:
1170                 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1171         case QM_MCR_RESULT_PENDING:
1172                 return "QM_MCR_RESULT_PENDING";
1173         case QM_MCR_RESULT_ERR_BADCOMMAND:
1174                 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1175         }
1176         return "<unknown MCR result>";
1177 }
1178
1179 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1180 {
1181         struct qm_fqd fqd;
1182         struct qm_mcr_queryfq_np np;
1183         struct qm_mc_command *mcc;
1184         struct qm_mc_result *mcr;
1185         struct qman_portal *p;
1186
1187         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1188                 int ret = qman_alloc_fqid(&fqid);
1189
1190                 if (ret)
1191                         return ret;
1192         }
1193         spin_lock_init(&fq->fqlock);
1194         fq->fqid = fqid;
1195         fq->flags = flags;
1196         fq->state = qman_fq_state_oos;
1197         fq->cgr_groupid = 0;
1198 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1199         if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1200                 pr_info("Find empty table entry failed\n");
1201                 return -ENOMEM;
1202         }
1203 #endif
1204         if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1205                 return 0;
1206         /* Everything else is AS_IS support */
1207         p = get_affine_portal();
1208         mcc = qm_mc_start(&p->p);
1209         mcc->queryfq.fqid = cpu_to_be32(fqid);
1210         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1211         while (!(mcr = qm_mc_result(&p->p)))
1212                 cpu_relax();
1213         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1214         if (mcr->result != QM_MCR_RESULT_OK) {
1215                 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1216                 goto err;
1217         }
1218         fqd = mcr->queryfq.fqd;
1219         hw_fqd_to_cpu(&fqd);
1220         mcc = qm_mc_start(&p->p);
1221         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1222         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1223         while (!(mcr = qm_mc_result(&p->p)))
1224                 cpu_relax();
1225         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1226         if (mcr->result != QM_MCR_RESULT_OK) {
1227                 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1228                 goto err;
1229         }
1230         np = mcr->queryfq_np;
1231         /* Phew, have queryfq and queryfq_np results, stitch together
1232          * the FQ object from those.
1233          */
1234         fq->cgr_groupid = fqd.cgid;
1235         switch (np.state & QM_MCR_NP_STATE_MASK) {
1236         case QM_MCR_NP_STATE_OOS:
1237                 break;
1238         case QM_MCR_NP_STATE_RETIRED:
1239                 fq->state = qman_fq_state_retired;
1240                 if (np.frm_cnt)
1241                         fq_set(fq, QMAN_FQ_STATE_NE);
1242                 break;
1243         case QM_MCR_NP_STATE_TEN_SCHED:
1244         case QM_MCR_NP_STATE_TRU_SCHED:
1245         case QM_MCR_NP_STATE_ACTIVE:
1246                 fq->state = qman_fq_state_sched;
1247                 if (np.state & QM_MCR_NP_STATE_R)
1248                         fq_set(fq, QMAN_FQ_STATE_CHANGING);
1249                 break;
1250         case QM_MCR_NP_STATE_PARKED:
1251                 fq->state = qman_fq_state_parked;
1252                 break;
1253         default:
1254                 DPAA_ASSERT(NULL == "invalid FQ state");
1255         }
1256         if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1257                 fq->state |= QMAN_FQ_STATE_CGR_EN;
1258         return 0;
1259 err:
1260         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1261                 qman_release_fqid(fqid);
1262         return -EIO;
1263 }
1264
1265 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1266 {
1267         /*
1268          * We don't need to lock the FQ as it is a pre-condition that the FQ be
1269          * quiesced. Instead, run some checks.
1270          */
1271         switch (fq->state) {
1272         case qman_fq_state_parked:
1273                 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1274                 /* Fallthrough */
1275         case qman_fq_state_oos:
1276                 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1277                         qman_release_fqid(fq->fqid);
1278 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1279                 clear_fq_table_entry(fq->key);
1280 #endif
1281                 return;
1282         default:
1283                 break;
1284         }
1285         DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1286 }
1287
1288 u32 qman_fq_fqid(struct qman_fq *fq)
1289 {
1290         return fq->fqid;
1291 }
1292
1293 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1294 {
1295         if (state)
1296                 *state = fq->state;
1297         if (flags)
1298                 *flags = fq->flags;
1299 }
1300
1301 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1302 {
1303         struct qm_mc_command *mcc;
1304         struct qm_mc_result *mcr;
1305         struct qman_portal *p;
1306
1307         u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1308                 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1309
1310         if ((fq->state != qman_fq_state_oos) &&
1311             (fq->state != qman_fq_state_parked))
1312                 return -EINVAL;
1313 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1314         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1315                 return -EINVAL;
1316 #endif
1317         if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1318                 /* And can't be set at the same time as TDTHRESH */
1319                 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1320                         return -EINVAL;
1321         }
1322         /* Issue an INITFQ_[PARKED|SCHED] management command */
1323         p = get_affine_portal();
1324         FQLOCK(fq);
1325         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1326                      ((fq->state != qman_fq_state_oos) &&
1327                                 (fq->state != qman_fq_state_parked)))) {
1328                 FQUNLOCK(fq);
1329                 return -EBUSY;
1330         }
1331         mcc = qm_mc_start(&p->p);
1332         if (opts)
1333                 mcc->initfq = *opts;
1334         mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1335         mcc->initfq.count = 0;
1336         /*
1337          * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1338          * demux pointer. Otherwise, the caller-provided value is allowed to
1339          * stand, don't overwrite it.
1340          */
1341         if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1342                 dma_addr_t phys_fq;
1343
1344                 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1345 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1346                 mcc->initfq.fqd.context_b = fq->key;
1347 #else
1348                 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1349 #endif
1350                 /*
1351                  *  and the physical address - NB, if the user wasn't trying to
1352                  * set CONTEXTA, clear the stashing settings.
1353                  */
1354                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1355                         mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1356                         memset(&mcc->initfq.fqd.context_a, 0,
1357                                sizeof(mcc->initfq.fqd.context_a));
1358                 } else {
1359                         phys_fq = rte_mem_virt2iova(fq);
1360                         qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1361                 }
1362         }
1363         if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1364                 mcc->initfq.fqd.dest.channel = p->config->channel;
1365                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1366                         mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1367                         mcc->initfq.fqd.dest.wq = 4;
1368                 }
1369         }
1370         mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1371         cpu_to_hw_fqd(&mcc->initfq.fqd);
1372         qm_mc_commit(&p->p, myverb);
1373         while (!(mcr = qm_mc_result(&p->p)))
1374                 cpu_relax();
1375         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1376         res = mcr->result;
1377         if (res != QM_MCR_RESULT_OK) {
1378                 FQUNLOCK(fq);
1379                 return -EIO;
1380         }
1381         if (opts) {
1382                 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1383                         if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1384                                 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1385                         else
1386                                 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1387                 }
1388                 if (opts->we_mask & QM_INITFQ_WE_CGID)
1389                         fq->cgr_groupid = opts->fqd.cgid;
1390         }
1391         fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1392                 qman_fq_state_sched : qman_fq_state_parked;
1393         FQUNLOCK(fq);
1394         return 0;
1395 }
1396
1397 int qman_schedule_fq(struct qman_fq *fq)
1398 {
1399         struct qm_mc_command *mcc;
1400         struct qm_mc_result *mcr;
1401         struct qman_portal *p;
1402
1403         int ret = 0;
1404         u8 res;
1405
1406         if (fq->state != qman_fq_state_parked)
1407                 return -EINVAL;
1408 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1409         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1410                 return -EINVAL;
1411 #endif
1412         /* Issue a ALTERFQ_SCHED management command */
1413         p = get_affine_portal();
1414
1415         FQLOCK(fq);
1416         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1417                      (fq->state != qman_fq_state_parked))) {
1418                 ret = -EBUSY;
1419                 goto out;
1420         }
1421         mcc = qm_mc_start(&p->p);
1422         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1423         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1424         while (!(mcr = qm_mc_result(&p->p)))
1425                 cpu_relax();
1426         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1427         res = mcr->result;
1428         if (res != QM_MCR_RESULT_OK) {
1429                 ret = -EIO;
1430                 goto out;
1431         }
1432         fq->state = qman_fq_state_sched;
1433 out:
1434         FQUNLOCK(fq);
1435
1436         return ret;
1437 }
1438
1439 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1440 {
1441         struct qm_mc_command *mcc;
1442         struct qm_mc_result *mcr;
1443         struct qman_portal *p;
1444
1445         int rval;
1446         u8 res;
1447
1448         if ((fq->state != qman_fq_state_parked) &&
1449             (fq->state != qman_fq_state_sched))
1450                 return -EINVAL;
1451 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1452         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1453                 return -EINVAL;
1454 #endif
1455         p = get_affine_portal();
1456
1457         FQLOCK(fq);
1458         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1459                      (fq->state == qman_fq_state_retired) ||
1460                                 (fq->state == qman_fq_state_oos))) {
1461                 rval = -EBUSY;
1462                 goto out;
1463         }
1464         rval = table_push_fq(p, fq);
1465         if (rval)
1466                 goto out;
1467         mcc = qm_mc_start(&p->p);
1468         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1469         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1470         while (!(mcr = qm_mc_result(&p->p)))
1471                 cpu_relax();
1472         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1473         res = mcr->result;
1474         /*
1475          * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1476          * and defer the flags until FQRNI or FQRN (respectively) show up. But
1477          * "Friendly" is to process OK immediately, and not set CHANGING. We do
1478          * friendly, otherwise the caller doesn't necessarily have a fully
1479          * "retired" FQ on return even if the retirement was immediate. However
1480          * this does mean some code duplication between here and
1481          * fq_state_change().
1482          */
1483         if (likely(res == QM_MCR_RESULT_OK)) {
1484                 rval = 0;
1485                 /* Process 'fq' right away, we'll ignore FQRNI */
1486                 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1487                         fq_set(fq, QMAN_FQ_STATE_NE);
1488                 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1489                         fq_set(fq, QMAN_FQ_STATE_ORL);
1490                 else
1491                         table_del_fq(p, fq);
1492                 if (flags)
1493                         *flags = fq->flags;
1494                 fq->state = qman_fq_state_retired;
1495                 if (fq->cb.fqs) {
1496                         /*
1497                          * Another issue with supporting "immediate" retirement
1498                          * is that we're forced to drop FQRNIs, because by the
1499                          * time they're seen it may already be "too late" (the
1500                          * fq may have been OOS'd and free()'d already). But if
1501                          * the upper layer wants a callback whether it's
1502                          * immediate or not, we have to fake a "MR" entry to
1503                          * look like an FQRNI...
1504                          */
1505                         struct qm_mr_entry msg;
1506
1507                         msg.verb = QM_MR_VERB_FQRNI;
1508                         msg.fq.fqs = mcr->alterfq.fqs;
1509                         msg.fq.fqid = fq->fqid;
1510 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1511                         msg.fq.contextB = fq->key;
1512 #else
1513                         msg.fq.contextB = (u32)(uintptr_t)fq;
1514 #endif
1515                         fq->cb.fqs(p, fq, &msg);
1516                 }
1517         } else if (res == QM_MCR_RESULT_PENDING) {
1518                 rval = 1;
1519                 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1520         } else {
1521                 rval = -EIO;
1522                 table_del_fq(p, fq);
1523         }
1524 out:
1525         FQUNLOCK(fq);
1526         return rval;
1527 }
1528
1529 int qman_oos_fq(struct qman_fq *fq)
1530 {
1531         struct qm_mc_command *mcc;
1532         struct qm_mc_result *mcr;
1533         struct qman_portal *p;
1534
1535         int ret = 0;
1536         u8 res;
1537
1538         if (fq->state != qman_fq_state_retired)
1539                 return -EINVAL;
1540 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1541         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1542                 return -EINVAL;
1543 #endif
1544         p = get_affine_portal();
1545         FQLOCK(fq);
1546         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1547                      (fq->state != qman_fq_state_retired))) {
1548                 ret = -EBUSY;
1549                 goto out;
1550         }
1551         mcc = qm_mc_start(&p->p);
1552         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1553         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1554         while (!(mcr = qm_mc_result(&p->p)))
1555                 cpu_relax();
1556         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1557         res = mcr->result;
1558         if (res != QM_MCR_RESULT_OK) {
1559                 ret = -EIO;
1560                 goto out;
1561         }
1562         fq->state = qman_fq_state_oos;
1563 out:
1564         FQUNLOCK(fq);
1565         return ret;
1566 }
1567
1568 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1569 {
1570         struct qm_mc_command *mcc;
1571         struct qm_mc_result *mcr;
1572         struct qman_portal *p;
1573
1574         int ret = 0;
1575         u8 res;
1576         u8 myverb;
1577
1578         if ((fq->state == qman_fq_state_oos) ||
1579             (fq->state == qman_fq_state_retired) ||
1580                 (fq->state == qman_fq_state_parked))
1581                 return -EINVAL;
1582
1583 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1584         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1585                 return -EINVAL;
1586 #endif
1587         /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1588         p = get_affine_portal();
1589         FQLOCK(fq);
1590         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1591                      (fq->state == qman_fq_state_parked) ||
1592                         (fq->state == qman_fq_state_oos) ||
1593                         (fq->state == qman_fq_state_retired))) {
1594                 ret = -EBUSY;
1595                 goto out;
1596         }
1597         mcc = qm_mc_start(&p->p);
1598         mcc->alterfq.fqid = fq->fqid;
1599         mcc->alterfq.count = 0;
1600         myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1601
1602         qm_mc_commit(&p->p, myverb);
1603         while (!(mcr = qm_mc_result(&p->p)))
1604                 cpu_relax();
1605         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1606
1607         res = mcr->result;
1608         if (res != QM_MCR_RESULT_OK) {
1609                 ret = -EIO;
1610                 goto out;
1611         }
1612 out:
1613         FQUNLOCK(fq);
1614         return ret;
1615 }
1616
1617 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1618 {
1619         struct qm_mc_command *mcc;
1620         struct qm_mc_result *mcr;
1621         struct qman_portal *p = get_affine_portal();
1622
1623         u8 res;
1624
1625         mcc = qm_mc_start(&p->p);
1626         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1627         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1628         while (!(mcr = qm_mc_result(&p->p)))
1629                 cpu_relax();
1630         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1631         res = mcr->result;
1632         if (res == QM_MCR_RESULT_OK)
1633                 *fqd = mcr->queryfq.fqd;
1634         hw_fqd_to_cpu(fqd);
1635         if (res != QM_MCR_RESULT_OK)
1636                 return -EIO;
1637         return 0;
1638 }
1639
1640 int qman_query_fq_has_pkts(struct qman_fq *fq)
1641 {
1642         struct qm_mc_command *mcc;
1643         struct qm_mc_result *mcr;
1644         struct qman_portal *p = get_affine_portal();
1645
1646         int ret = 0;
1647         u8 res;
1648
1649         mcc = qm_mc_start(&p->p);
1650         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1651         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1652         while (!(mcr = qm_mc_result(&p->p)))
1653                 cpu_relax();
1654         res = mcr->result;
1655         if (res == QM_MCR_RESULT_OK)
1656                 ret = !!mcr->queryfq_np.frm_cnt;
1657         return ret;
1658 }
1659
1660 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1661 {
1662         struct qm_mc_command *mcc;
1663         struct qm_mc_result *mcr;
1664         struct qman_portal *p = get_affine_portal();
1665
1666         u8 res;
1667
1668         mcc = qm_mc_start(&p->p);
1669         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1670         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1671         while (!(mcr = qm_mc_result(&p->p)))
1672                 cpu_relax();
1673         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1674         res = mcr->result;
1675         if (res == QM_MCR_RESULT_OK) {
1676                 *np = mcr->queryfq_np;
1677                 np->fqd_link = be24_to_cpu(np->fqd_link);
1678                 np->odp_seq = be16_to_cpu(np->odp_seq);
1679                 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1680                 np->orp_ea_hseq  = be16_to_cpu(np->orp_ea_hseq);
1681                 np->orp_ea_tseq  = be16_to_cpu(np->orp_ea_tseq);
1682                 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1683                 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1684                 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1685                 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1686                 np->ics_surp = be16_to_cpu(np->ics_surp);
1687                 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1688                 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1689                 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1690                 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1691                 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1692                 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1693                 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1694         }
1695         if (res == QM_MCR_RESULT_ERR_FQID)
1696                 return -ERANGE;
1697         else if (res != QM_MCR_RESULT_OK)
1698                 return -EIO;
1699         return 0;
1700 }
1701
1702 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1703 {
1704         struct qm_mc_command *mcc;
1705         struct qm_mc_result *mcr;
1706         struct qman_portal *p = get_affine_portal();
1707
1708         u8 res, myverb;
1709
1710         myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1711                                  QM_MCR_VERB_QUERYWQ;
1712         mcc = qm_mc_start(&p->p);
1713         mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1714         qm_mc_commit(&p->p, myverb);
1715         while (!(mcr = qm_mc_result(&p->p)))
1716                 cpu_relax();
1717         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1718         res = mcr->result;
1719         if (res == QM_MCR_RESULT_OK) {
1720                 int i, array_len;
1721
1722                 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1723                 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1724                 for (i = 0; i < array_len; i++)
1725                         wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1726         }
1727         if (res != QM_MCR_RESULT_OK) {
1728                 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1729                 return -EIO;
1730         }
1731         return 0;
1732 }
1733
1734 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1735                        struct qm_mcr_cgrtestwrite *result)
1736 {
1737         struct qm_mc_command *mcc;
1738         struct qm_mc_result *mcr;
1739         struct qman_portal *p = get_affine_portal();
1740
1741         u8 res;
1742
1743         mcc = qm_mc_start(&p->p);
1744         mcc->cgrtestwrite.cgid = cgr->cgrid;
1745         mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1746         mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1747         qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1748         while (!(mcr = qm_mc_result(&p->p)))
1749                 cpu_relax();
1750         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1751         res = mcr->result;
1752         if (res == QM_MCR_RESULT_OK)
1753                 *result = mcr->cgrtestwrite;
1754         if (res != QM_MCR_RESULT_OK) {
1755                 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1756                 return -EIO;
1757         }
1758         return 0;
1759 }
1760
1761 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1762 {
1763         struct qm_mc_command *mcc;
1764         struct qm_mc_result *mcr;
1765         struct qman_portal *p = get_affine_portal();
1766         u8 res;
1767         unsigned int i;
1768
1769         mcc = qm_mc_start(&p->p);
1770         mcc->querycgr.cgid = cgr->cgrid;
1771         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
1772         while (!(mcr = qm_mc_result(&p->p)))
1773                 cpu_relax();
1774         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
1775         res = mcr->result;
1776         if (res == QM_MCR_RESULT_OK)
1777                 *cgrd = mcr->querycgr;
1778         if (res != QM_MCR_RESULT_OK) {
1779                 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
1780                 return -EIO;
1781         }
1782         cgrd->cgr.wr_parm_g.word =
1783                 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
1784         cgrd->cgr.wr_parm_y.word =
1785                 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
1786         cgrd->cgr.wr_parm_r.word =
1787                 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
1788         cgrd->cgr.cscn_targ =  be32_to_cpu(cgrd->cgr.cscn_targ);
1789         cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
1790         for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
1791                 cgrd->cscn_targ_swp[i] =
1792                         be32_to_cpu(cgrd->cscn_targ_swp[i]);
1793         return 0;
1794 }
1795
1796 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
1797 {
1798         struct qm_mc_result *mcr;
1799         struct qman_portal *p = get_affine_portal();
1800         u8 res;
1801         unsigned int i;
1802
1803         qm_mc_start(&p->p);
1804         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
1805         while (!(mcr = qm_mc_result(&p->p)))
1806                 cpu_relax();
1807         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
1808                         QM_MCC_VERB_QUERYCONGESTION);
1809         res = mcr->result;
1810         if (res == QM_MCR_RESULT_OK)
1811                 *congestion = mcr->querycongestion;
1812         if (res != QM_MCR_RESULT_OK) {
1813                 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
1814                 return -EIO;
1815         }
1816         for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
1817                 congestion->state.state[i] =
1818                         be32_to_cpu(congestion->state.state[i]);
1819         return 0;
1820 }
1821
1822 int qman_set_vdq(struct qman_fq *fq, u16 num)
1823 {
1824         struct qman_portal *p = get_affine_portal();
1825         uint32_t vdqcr;
1826         int ret = -EBUSY;
1827
1828         vdqcr = QM_VDQCR_EXACT;
1829         vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
1830
1831         if ((fq->state != qman_fq_state_parked) &&
1832             (fq->state != qman_fq_state_retired)) {
1833                 ret = -EINVAL;
1834                 goto out;
1835         }
1836         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
1837                 ret = -EBUSY;
1838                 goto out;
1839         }
1840         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1841
1842         if (!p->vdqcr_owned) {
1843                 FQLOCK(fq);
1844                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1845                         goto escape;
1846                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1847                 FQUNLOCK(fq);
1848                 p->vdqcr_owned = fq;
1849                 ret = 0;
1850         }
1851 escape:
1852         if (!ret)
1853                 qm_dqrr_vdqcr_set(&p->p, vdqcr);
1854
1855 out:
1856         return ret;
1857 }
1858
1859 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
1860                           u32 vdqcr)
1861 {
1862         struct qman_portal *p;
1863         int ret = -EBUSY;
1864
1865         if ((fq->state != qman_fq_state_parked) &&
1866             (fq->state != qman_fq_state_retired))
1867                 return -EINVAL;
1868         if (vdqcr & QM_VDQCR_FQID_MASK)
1869                 return -EINVAL;
1870         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1871                 return -EBUSY;
1872         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1873
1874         p = get_affine_portal();
1875
1876         if (!p->vdqcr_owned) {
1877                 FQLOCK(fq);
1878                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1879                         goto escape;
1880                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1881                 FQUNLOCK(fq);
1882                 p->vdqcr_owned = fq;
1883                 ret = 0;
1884         }
1885 escape:
1886         if (ret)
1887                 return ret;
1888
1889         /* VDQCR is set */
1890         qm_dqrr_vdqcr_set(&p->p, vdqcr);
1891         return 0;
1892 }
1893
1894 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
1895 {
1896         if (avail)
1897                 qm_eqcr_cce_prefetch(&p->p);
1898         else
1899                 qm_eqcr_cce_update(&p->p);
1900 }
1901
1902 int qman_eqcr_is_empty(void)
1903 {
1904         struct qman_portal *p = get_affine_portal();
1905         u8 avail;
1906
1907         update_eqcr_ci(p, 0);
1908         avail = qm_eqcr_get_fill(&p->p);
1909         return (avail == 0);
1910 }
1911
1912 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
1913 {
1914         if (affine) {
1915                 struct qman_portal *p = get_affine_portal();
1916
1917                 p->cb_dc_ern = handler;
1918         } else
1919                 cb_dc_ern = handler;
1920 }
1921
1922 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
1923                                         struct qman_fq *fq,
1924                                         const struct qm_fd *fd,
1925                                         u32 flags)
1926 {
1927         struct qm_eqcr_entry *eq;
1928         u8 avail;
1929
1930         if (p->use_eqcr_ci_stashing) {
1931                 /*
1932                  * The stashing case is easy, only update if we need to in
1933                  * order to try and liberate ring entries.
1934                  */
1935                 eq = qm_eqcr_start_stash(&p->p);
1936         } else {
1937                 /*
1938                  * The non-stashing case is harder, need to prefetch ahead of
1939                  * time.
1940                  */
1941                 avail = qm_eqcr_get_avail(&p->p);
1942                 if (avail < 2)
1943                         update_eqcr_ci(p, avail);
1944                 eq = qm_eqcr_start_no_stash(&p->p);
1945         }
1946
1947         if (unlikely(!eq))
1948                 return NULL;
1949
1950         if (flags & QMAN_ENQUEUE_FLAG_DCA)
1951                 eq->dca = QM_EQCR_DCA_ENABLE |
1952                         ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
1953                                         QM_EQCR_DCA_PARK : 0) |
1954                         ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
1955         eq->fqid = cpu_to_be32(fq->fqid);
1956 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1957         eq->tag = cpu_to_be32(fq->key);
1958 #else
1959         eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
1960 #endif
1961         eq->fd = *fd;
1962         cpu_to_hw_fd(&eq->fd);
1963         return eq;
1964 }
1965
1966 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
1967 {
1968         struct qman_portal *p = get_affine_portal();
1969         struct qm_eqcr_entry *eq;
1970
1971         eq = try_p_eq_start(p, fq, fd, flags);
1972         if (!eq)
1973                 return -EBUSY;
1974         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
1975         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
1976                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
1977         /* Factor the below out, it's used from qman_enqueue_orp() too */
1978         return 0;
1979 }
1980
1981 int qman_enqueue_multi(struct qman_fq *fq,
1982                        const struct qm_fd *fd,
1983                 int frames_to_send)
1984 {
1985         struct qman_portal *p = get_affine_portal();
1986         struct qm_portal *portal = &p->p;
1987
1988         register struct qm_eqcr *eqcr = &portal->eqcr;
1989         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
1990
1991         u8 i, diff, old_ci, sent = 0;
1992
1993         /* Update the available entries if no entry is free */
1994         if (!eqcr->available) {
1995                 old_ci = eqcr->ci;
1996                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
1997                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
1998                 eqcr->available += diff;
1999                 if (!diff)
2000                         return 0;
2001         }
2002
2003         /* try to send as many frames as possible */
2004         while (eqcr->available && frames_to_send--) {
2005                 eq->fqid = cpu_to_be32(fq->fqid);
2006 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2007                 eq->tag = cpu_to_be32(fq->key);
2008 #else
2009                 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2010 #endif
2011                 eq->fd.opaque_addr = fd->opaque_addr;
2012                 eq->fd.addr = cpu_to_be40(fd->addr);
2013                 eq->fd.status = cpu_to_be32(fd->status);
2014                 eq->fd.opaque = cpu_to_be32(fd->opaque);
2015
2016                 eq = (void *)((unsigned long)(eq + 1) &
2017                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2018                 eqcr->available--;
2019                 sent++;
2020                 fd++;
2021         }
2022         lwsync();
2023
2024         /* In order for flushes to complete faster, all lines are recorded in
2025          * 32 bit word.
2026          */
2027         eq = eqcr->cursor;
2028         for (i = 0; i < sent; i++) {
2029                 eq->__dont_write_directly__verb =
2030                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2031                 prev_eq = eq;
2032                 eq = (void *)((unsigned long)(eq + 1) &
2033                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2034                 if (unlikely((prev_eq + 1) != eq))
2035                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2036         }
2037
2038         /* We need  to flush all the lines but without load/store operations
2039          * between them
2040          */
2041         eq = eqcr->cursor;
2042         for (i = 0; i < sent; i++) {
2043                 dcbf(eq);
2044                 eq = (void *)((unsigned long)(eq + 1) &
2045                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2046         }
2047         /* Update cursor for the next call */
2048         eqcr->cursor = eq;
2049         return sent;
2050 }
2051
2052 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2053                      struct qman_fq *orp, u16 orp_seqnum)
2054 {
2055         struct qman_portal *p  = get_affine_portal();
2056         struct qm_eqcr_entry *eq;
2057
2058         eq = try_p_eq_start(p, fq, fd, flags);
2059         if (!eq)
2060                 return -EBUSY;
2061         /* Process ORP-specifics here */
2062         if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2063                 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2064         else {
2065                 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2066                 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2067                         orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2068                 else
2069                         /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2070                         orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2071         }
2072         eq->seqnum = cpu_to_be16(orp_seqnum);
2073         eq->orp = cpu_to_be32(orp->fqid);
2074         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2075         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2076                 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2077                                 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2078                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2079
2080         return 0;
2081 }
2082
2083 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2084                     struct qm_mcc_initcgr *opts)
2085 {
2086         struct qm_mc_command *mcc;
2087         struct qm_mc_result *mcr;
2088         struct qman_portal *p = get_affine_portal();
2089
2090         u8 res;
2091         u8 verb = QM_MCC_VERB_MODIFYCGR;
2092
2093         mcc = qm_mc_start(&p->p);
2094         if (opts)
2095                 mcc->initcgr = *opts;
2096         mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2097         mcc->initcgr.cgr.wr_parm_g.word =
2098                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2099         mcc->initcgr.cgr.wr_parm_y.word =
2100                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2101         mcc->initcgr.cgr.wr_parm_r.word =
2102                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2103         mcc->initcgr.cgr.cscn_targ =  cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2104         mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2105
2106         mcc->initcgr.cgid = cgr->cgrid;
2107         if (flags & QMAN_CGR_FLAG_USE_INIT)
2108                 verb = QM_MCC_VERB_INITCGR;
2109         qm_mc_commit(&p->p, verb);
2110         while (!(mcr = qm_mc_result(&p->p)))
2111                 cpu_relax();
2112
2113         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2114         res = mcr->result;
2115         return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2116 }
2117
2118 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2119                                         QM_CHANNEL_SWPORTAL0))
2120 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2121 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2122
2123 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2124                     struct qm_mcc_initcgr *opts)
2125 {
2126         struct qm_mcr_querycgr cgr_state;
2127         struct qm_mcc_initcgr local_opts;
2128         int ret;
2129         struct qman_portal *p;
2130
2131         /* We have to check that the provided CGRID is within the limits of the
2132          * data-structures, for obvious reasons. However we'll let h/w take
2133          * care of determining whether it's within the limits of what exists on
2134          * the SoC.
2135          */
2136         if (cgr->cgrid >= __CGR_NUM)
2137                 return -EINVAL;
2138
2139         p = get_affine_portal();
2140
2141         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2142         cgr->chan = p->config->channel;
2143         spin_lock(&p->cgr_lock);
2144
2145         /* if no opts specified, just add it to the list */
2146         if (!opts)
2147                 goto add_list;
2148
2149         ret = qman_query_cgr(cgr, &cgr_state);
2150         if (ret)
2151                 goto release_lock;
2152         if (opts)
2153                 local_opts = *opts;
2154         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2155                 local_opts.cgr.cscn_targ_upd_ctrl =
2156                         QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2157         else
2158                 /* Overwrite TARG */
2159                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2160                                                         TARG_MASK(p);
2161         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2162
2163         /* send init if flags indicate so */
2164         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2165                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2166         else
2167                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2168         if (ret)
2169                 goto release_lock;
2170 add_list:
2171         list_add(&cgr->node, &p->cgr_cbs);
2172
2173         /* Determine if newly added object requires its callback to be called */
2174         ret = qman_query_cgr(cgr, &cgr_state);
2175         if (ret) {
2176                 /* we can't go back, so proceed and return success, but screen
2177                  * and wail to the log file.
2178                  */
2179                 pr_crit("CGR HW state partially modified\n");
2180                 ret = 0;
2181                 goto release_lock;
2182         }
2183         if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2184                                                               cgr->cgrid))
2185                 cgr->cb(p, cgr, 1);
2186 release_lock:
2187         spin_unlock(&p->cgr_lock);
2188         return ret;
2189 }
2190
2191 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2192                            struct qm_mcc_initcgr *opts)
2193 {
2194         struct qm_mcc_initcgr local_opts;
2195         struct qm_mcr_querycgr cgr_state;
2196         int ret;
2197
2198         if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2199                 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2200                 return -EINVAL;
2201         }
2202         /* We have to check that the provided CGRID is within the limits of the
2203          * data-structures, for obvious reasons. However we'll let h/w take
2204          * care of determining whether it's within the limits of what exists on
2205          * the SoC.
2206          */
2207         if (cgr->cgrid >= __CGR_NUM)
2208                 return -EINVAL;
2209
2210         ret = qman_query_cgr(cgr, &cgr_state);
2211         if (ret)
2212                 return ret;
2213
2214         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2215         if (opts)
2216                 local_opts = *opts;
2217
2218         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2219                 local_opts.cgr.cscn_targ_upd_ctrl =
2220                                 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2221                                 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2222         else
2223                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2224                                         TARG_DCP_MASK(dcp_portal);
2225         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2226
2227         /* send init if flags indicate so */
2228         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2229                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2230                                       &local_opts);
2231         else
2232                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2233
2234         return ret;
2235 }
2236
2237 int qman_delete_cgr(struct qman_cgr *cgr)
2238 {
2239         struct qm_mcr_querycgr cgr_state;
2240         struct qm_mcc_initcgr local_opts;
2241         int ret = 0;
2242         struct qman_cgr *i;
2243         struct qman_portal *p = get_affine_portal();
2244
2245         if (cgr->chan != p->config->channel) {
2246                 pr_crit("Attempting to delete cgr from different portal than"
2247                         " it was create: create 0x%x, delete 0x%x\n",
2248                         cgr->chan, p->config->channel);
2249                 ret = -EINVAL;
2250                 goto put_portal;
2251         }
2252         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2253         spin_lock(&p->cgr_lock);
2254         list_del(&cgr->node);
2255         /*
2256          * If there are no other CGR objects for this CGRID in the list,
2257          * update CSCN_TARG accordingly
2258          */
2259         list_for_each_entry(i, &p->cgr_cbs, node)
2260                 if ((i->cgrid == cgr->cgrid) && i->cb)
2261                         goto release_lock;
2262         ret = qman_query_cgr(cgr, &cgr_state);
2263         if (ret)  {
2264                 /* add back to the list */
2265                 list_add(&cgr->node, &p->cgr_cbs);
2266                 goto release_lock;
2267         }
2268         /* Overwrite TARG */
2269         local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2270         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2271                 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2272         else
2273                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2274                                                          ~(TARG_MASK(p));
2275         ret = qman_modify_cgr(cgr, 0, &local_opts);
2276         if (ret)
2277                 /* add back to the list */
2278                 list_add(&cgr->node, &p->cgr_cbs);
2279 release_lock:
2280         spin_unlock(&p->cgr_lock);
2281 put_portal:
2282         return ret;
2283 }
2284
2285 int qman_shutdown_fq(u32 fqid)
2286 {
2287         struct qman_portal *p;
2288         struct qm_portal *low_p;
2289         struct qm_mc_command *mcc;
2290         struct qm_mc_result *mcr;
2291         u8 state;
2292         int orl_empty, fq_empty, drain = 0;
2293         u32 result;
2294         u32 channel, wq;
2295         u16 dest_wq;
2296
2297         p = get_affine_portal();
2298         low_p = &p->p;
2299
2300         /* Determine the state of the FQID */
2301         mcc = qm_mc_start(low_p);
2302         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2303         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2304         while (!(mcr = qm_mc_result(low_p)))
2305                 cpu_relax();
2306         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2307         state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2308         if (state == QM_MCR_NP_STATE_OOS)
2309                 return 0; /* Already OOS, no need to do anymore checks */
2310
2311         /* Query which channel the FQ is using */
2312         mcc = qm_mc_start(low_p);
2313         mcc->queryfq.fqid = cpu_to_be32(fqid);
2314         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2315         while (!(mcr = qm_mc_result(low_p)))
2316                 cpu_relax();
2317         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2318
2319         /* Need to store these since the MCR gets reused */
2320         dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2321         channel = dest_wq & 0x7;
2322         wq = dest_wq >> 3;
2323
2324         switch (state) {
2325         case QM_MCR_NP_STATE_TEN_SCHED:
2326         case QM_MCR_NP_STATE_TRU_SCHED:
2327         case QM_MCR_NP_STATE_ACTIVE:
2328         case QM_MCR_NP_STATE_PARKED:
2329                 orl_empty = 0;
2330                 mcc = qm_mc_start(low_p);
2331                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2332                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2333                 while (!(mcr = qm_mc_result(low_p)))
2334                         cpu_relax();
2335                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2336                            QM_MCR_VERB_ALTER_RETIRE);
2337                 result = mcr->result; /* Make a copy as we reuse MCR below */
2338
2339                 if (result == QM_MCR_RESULT_PENDING) {
2340                         /* Need to wait for the FQRN in the message ring, which
2341                          * will only occur once the FQ has been drained.  In
2342                          * order for the FQ to drain the portal needs to be set
2343                          * to dequeue from the channel the FQ is scheduled on
2344                          */
2345                         const struct qm_mr_entry *msg;
2346                         const struct qm_dqrr_entry *dqrr = NULL;
2347                         int found_fqrn = 0;
2348                         __maybe_unused u16 dequeue_wq = 0;
2349
2350                         /* Flag that we need to drain FQ */
2351                         drain = 1;
2352
2353                         if (channel >= qm_channel_pool1 &&
2354                             channel < (u16)(qm_channel_pool1 + 15)) {
2355                                 /* Pool channel, enable the bit in the portal */
2356                                 dequeue_wq = (channel -
2357                                               qm_channel_pool1 + 1) << 4 | wq;
2358                         } else if (channel < qm_channel_pool1) {
2359                                 /* Dedicated channel */
2360                                 dequeue_wq = wq;
2361                         } else {
2362                                 pr_info("Cannot recover FQ 0x%x,"
2363                                         " it is scheduled on channel 0x%x",
2364                                         fqid, channel);
2365                                 return -EBUSY;
2366                         }
2367                         /* Set the sdqcr to drain this channel */
2368                         if (channel < qm_channel_pool1)
2369                                 qm_dqrr_sdqcr_set(low_p,
2370                                                   QM_SDQCR_TYPE_ACTIVE |
2371                                           QM_SDQCR_CHANNELS_DEDICATED);
2372                         else
2373                                 qm_dqrr_sdqcr_set(low_p,
2374                                                   QM_SDQCR_TYPE_ACTIVE |
2375                                                   QM_SDQCR_CHANNELS_POOL_CONV
2376                                                   (channel));
2377                         while (!found_fqrn) {
2378                                 /* Keep draining DQRR while checking the MR*/
2379                                 qm_dqrr_pvb_update(low_p);
2380                                 dqrr = qm_dqrr_current(low_p);
2381                                 while (dqrr) {
2382                                         qm_dqrr_cdc_consume_1ptr(
2383                                                 low_p, dqrr, 0);
2384                                         qm_dqrr_pvb_update(low_p);
2385                                         qm_dqrr_next(low_p);
2386                                         dqrr = qm_dqrr_current(low_p);
2387                                 }
2388                                 /* Process message ring too */
2389                                 qm_mr_pvb_update(low_p);
2390                                 msg = qm_mr_current(low_p);
2391                                 while (msg) {
2392                                         if ((msg->verb &
2393                                              QM_MR_VERB_TYPE_MASK)
2394                                             == QM_MR_VERB_FQRN)
2395                                                 found_fqrn = 1;
2396                                         qm_mr_next(low_p);
2397                                         qm_mr_cci_consume_to_current(low_p);
2398                                         qm_mr_pvb_update(low_p);
2399                                         msg = qm_mr_current(low_p);
2400                                 }
2401                                 cpu_relax();
2402                         }
2403                 }
2404                 if (result != QM_MCR_RESULT_OK &&
2405                     result !=  QM_MCR_RESULT_PENDING) {
2406                         /* error */
2407                         pr_err("qman_retire_fq failed on FQ 0x%x,"
2408                                " result=0x%x\n", fqid, result);
2409                         return -1;
2410                 }
2411                 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2412                         /* ORL had no entries, no need to wait until the
2413                          * ERNs come in.
2414                          */
2415                         orl_empty = 1;
2416                 }
2417                 /* Retirement succeeded, check to see if FQ needs
2418                  * to be drained.
2419                  */
2420                 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2421                         /* FQ is Not Empty, drain using volatile DQ commands */
2422                         fq_empty = 0;
2423                         do {
2424                                 const struct qm_dqrr_entry *dqrr = NULL;
2425                                 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2426
2427                                 qm_dqrr_vdqcr_set(low_p, vdqcr);
2428
2429                                 /* Wait for a dequeue to occur */
2430                                 while (dqrr == NULL) {
2431                                         qm_dqrr_pvb_update(low_p);
2432                                         dqrr = qm_dqrr_current(low_p);
2433                                         if (!dqrr)
2434                                                 cpu_relax();
2435                                 }
2436                                 /* Process the dequeues, making sure to
2437                                  * empty the ring completely.
2438                                  */
2439                                 while (dqrr) {
2440                                         if (dqrr->fqid == fqid &&
2441                                             dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2442                                                 fq_empty = 1;
2443                                         qm_dqrr_cdc_consume_1ptr(low_p,
2444                                                                  dqrr, 0);
2445                                         qm_dqrr_pvb_update(low_p);
2446                                         qm_dqrr_next(low_p);
2447                                         dqrr = qm_dqrr_current(low_p);
2448                                 }
2449                         } while (fq_empty == 0);
2450                 }
2451                 qm_dqrr_sdqcr_set(low_p, 0);
2452
2453                 /* Wait for the ORL to have been completely drained */
2454                 while (orl_empty == 0) {
2455                         const struct qm_mr_entry *msg;
2456
2457                         qm_mr_pvb_update(low_p);
2458                         msg = qm_mr_current(low_p);
2459                         while (msg) {
2460                                 if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
2461                                     QM_MR_VERB_FQRL)
2462                                         orl_empty = 1;
2463                                 qm_mr_next(low_p);
2464                                 qm_mr_cci_consume_to_current(low_p);
2465                                 qm_mr_pvb_update(low_p);
2466                                 msg = qm_mr_current(low_p);
2467                         }
2468                         cpu_relax();
2469                 }
2470                 mcc = qm_mc_start(low_p);
2471                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2472                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2473                 while (!(mcr = qm_mc_result(low_p)))
2474                         cpu_relax();
2475                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2476                            QM_MCR_VERB_ALTER_OOS);
2477                 if (mcr->result != QM_MCR_RESULT_OK) {
2478                         pr_err(
2479                         "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2480                                fqid, mcr->result);
2481                         return -1;
2482                 }
2483                 return 0;
2484
2485         case QM_MCR_NP_STATE_RETIRED:
2486                 /* Send OOS Command */
2487                 mcc = qm_mc_start(low_p);
2488                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2489                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2490                 while (!(mcr = qm_mc_result(low_p)))
2491                         cpu_relax();
2492                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2493                            QM_MCR_VERB_ALTER_OOS);
2494                 if (mcr->result) {
2495                         pr_err("OOS Failed on FQID 0x%x\n", fqid);
2496                         return -1;
2497                 }
2498                 return 0;
2499
2500         }
2501         return -1;
2502 }