drivers/bus/fslmc/qbman/qbman_portal.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
   4  *
   5  */
   6
   7 #include "qbman_portal.h"
   8
   9 /* QBMan portal management command codes */
  10 #define QBMAN_MC_ACQUIRE       0x30
  11 #define QBMAN_WQCHAN_CONFIGURE 0x46
  12
  13 /* CINH register offsets */
  14 #define QBMAN_CINH_SWP_EQCR_PI 0x800
  15 #define QBMAN_CINH_SWP_EQCR_CI 0x840
  16 #define QBMAN_CINH_SWP_EQAR    0x8c0
  17 #define QBMAN_CINH_SWP_DQPI    0xa00
  18 #define QBMAN_CINH_SWP_DCAP    0xac0
  19 #define QBMAN_CINH_SWP_SDQCR   0xb00
  20 #define QBMAN_CINH_SWP_RAR     0xcc0
  21 #define QBMAN_CINH_SWP_ISR     0xe00
  22 #define QBMAN_CINH_SWP_IER     0xe40
  23 #define QBMAN_CINH_SWP_ISDR    0xe80
  24 #define QBMAN_CINH_SWP_IIR     0xec0
  25 #define QBMAN_CINH_SWP_DQRR_ITR    0xa80
  26 #define QBMAN_CINH_SWP_ITPR    0xf40
  27
  28 /* CENA register offsets */
  29 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((uint32_t)(n) << 6))
  30 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((uint32_t)(n) << 6))
  31 #define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((uint32_t)(n) << 6))
  32 #define QBMAN_CENA_SWP_CR      0x600
  33 #define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((uint32_t)(vb) >> 1))
  34 #define QBMAN_CENA_SWP_VDQCR   0x780
  35 #define QBMAN_CENA_SWP_EQCR_CI 0x840
  36
  37 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
  38 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6)
  39
  40 /* QBMan FQ management command codes */
  41 #define QBMAN_FQ_SCHEDULE       0x48
  42 #define QBMAN_FQ_FORCE          0x49
  43 #define QBMAN_FQ_XON            0x4d
  44 #define QBMAN_FQ_XOFF           0x4e
  45
  46 /*******************************/
  47 /* Pre-defined attribute codes */
  48 /*******************************/
  49
  50 #define QBMAN_RESPONSE_VERB_MASK   0x7f
  51
  52 /*************************/
  53 /* SDQCR attribute codes */
  54 /*************************/
  55 #define QB_SDQCR_FC_SHIFT   29
  56 #define QB_SDQCR_FC_MASK    0x1
  57 #define QB_SDQCR_DCT_SHIFT  24
  58 #define QB_SDQCR_DCT_MASK   0x3
  59 #define QB_SDQCR_TOK_SHIFT  16
  60 #define QB_SDQCR_TOK_MASK   0xff
  61 #define QB_SDQCR_SRC_SHIFT  0
  62 #define QB_SDQCR_SRC_MASK   0xffff
  63
  64 /* opaque token for static dequeues */
  65 #define QMAN_SDQCR_TOKEN    0xbb
  66
  67 enum qbman_sdqcr_dct {
  68         qbman_sdqcr_dct_null = 0,
  69         qbman_sdqcr_dct_prio_ics,
  70         qbman_sdqcr_dct_active_ics,
  71         qbman_sdqcr_dct_active
  72 };
  73
  74 enum qbman_sdqcr_fc {
  75         qbman_sdqcr_fc_one = 0,
  76         qbman_sdqcr_fc_up_to_3 = 1
  77 };
  78
  79 /* We need to keep track of which SWP triggered a pull command
  80  * so keep an array of portal IDs and use the token field to
  81  * be able to find the proper portal
  82  */
  83 #define MAX_QBMAN_PORTALS  64
  84 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS];
  85
  86 /*********************************/
  87 /* Portal constructor/destructor */
  88 /*********************************/
  89
  90 /* Software portals should always be in the power-on state when we initialise,
  91  * due to the CCSR-based portal reset functionality that MC has.
  92  *
  93  * Erk! Turns out that QMan versions prior to 4.1 do not correctly reset DQRR
  94  * valid-bits, so we need to support a workaround where we don't trust
  95  * valid-bits when detecting new entries until any stale ring entries have been
  96  * overwritten at least once. The idea is that we read PI for the first few
  97  * entries, then switch to valid-bit after that. The trick is to clear the
  98  * bug-work-around boolean once the PI wraps around the ring for the first time.
  99  *
 100  * Note: this still carries a slight additional cost once the decrementer hits
 101  * zero.
 102  */
 103 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 104 {
 105         int ret;
 106         uint32_t eqcr_pi;
 107         struct qbman_swp *p = malloc(sizeof(*p));
 108
 109         if (!p)
 110                 return NULL;
 111         p->desc = *d;
 112 #ifdef QBMAN_CHECKING
 113         p->mc.check = swp_mc_can_start;
 114 #endif
 115         p->mc.valid_bit = QB_VALID_BIT;
 116         p->sdq = 0;
 117         p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
 118         p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
 119         p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
 120
 121         atomic_set(&p->vdq.busy, 1);
 122         p->vdq.valid_bit = QB_VALID_BIT;
 123         p->dqrr.next_idx = 0;
 124         p->dqrr.valid_bit = QB_VALID_BIT;
 125         if ((p->desc.qman_version & 0xFFFF0000) < QMAN_REV_4100) {
 126                 p->dqrr.dqrr_size = 4;
 127                 p->dqrr.reset_bug = 1;
 128         } else {
 129                 p->dqrr.dqrr_size = 8;
 130                 p->dqrr.reset_bug = 0;
 131         }
 132
 133         ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size);
 134         if (ret) {
 135                 free(p);
 136                 pr_err("qbman_swp_sys_init() failed %d\n", ret);
 137                 return NULL;
 138         }
 139         /* SDQCR needs to be initialized to 0 when no channels are
 140          * being dequeued from or else the QMan HW will indicate an
 141          * error.  The values that were calculated above will be
 142          * applied when dequeues from a specific channel are enabled.
 143          */
 144         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0);
 145         eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI);
 146         p->eqcr.pi = eqcr_pi & 0xF;
 147         p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
 148         p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) & 0xF;
 149         p->eqcr.available = QBMAN_EQCR_SIZE - qm_cyc_diff(QBMAN_EQCR_SIZE,
 150                                                 p->eqcr.ci, p->eqcr.pi);
 151
 152         portal_idx_map[p->desc.idx] = p;
 153         return p;
 154 }
 155
 156 void qbman_swp_finish(struct qbman_swp *p)
 157 {
 158 #ifdef QBMAN_CHECKING
 159         QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
 160 #endif
 161         qbman_swp_sys_finish(&p->sys);
 162         portal_idx_map[p->desc.idx] = NULL;
 163         free(p);
 164 }
 165
 166 const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p)
 167 {
 168         return &p->desc;
 169 }
 170
 171 /**************/
 172 /* Interrupts */
 173 /**************/
 174
 175 uint32_t qbman_swp_interrupt_get_vanish(struct qbman_swp *p)
 176 {
 177         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISDR);
 178 }
 179
 180 void qbman_swp_interrupt_set_vanish(struct qbman_swp *p, uint32_t mask)
 181 {
 182         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISDR, mask);
 183 }
 184
 185 uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p)
 186 {
 187         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISR);
 188 }
 189
 190 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask)
 191 {
 192         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask);
 193 }
 194
 195 uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p)
 196 {
 197         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR);
 198 }
 199
 200 void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask)
 201 {
 202         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask);
 203 }
 204
 205 uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p)
 206 {
 207         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR);
 208 }
 209
 210 void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask)
 211 {
 212         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask);
 213 }
 214
 215 uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
 216 {
 217         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER);
 218 }
 219
 220 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, uint32_t mask)
 221 {
 222         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IER, mask);
 223 }
 224
 225 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
 226 {
 227         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IIR);
 228 }
 229
 230 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
 231 {
 232         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
 233 }
 234
 235 /***********************/
 236 /* Management commands */
 237 /***********************/
 238
 239 /*
 240  * Internal code common to all types of management commands.
 241  */
 242
 243 void *qbman_swp_mc_start(struct qbman_swp *p)
 244 {
 245         void *ret;
 246 #ifdef QBMAN_CHECKING
 247         QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
 248 #endif
 249         ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
 250 #ifdef QBMAN_CHECKING
 251         if (!ret)
 252                 p->mc.check = swp_mc_can_submit;
 253 #endif
 254         return ret;
 255 }
 256
 257 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb)
 258 {
 259         uint8_t *v = cmd;
 260 #ifdef QBMAN_CHECKING
 261         QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit));
 262 #endif
 263         /* TBD: "|=" is going to hurt performance. Need to move as many fields
 264          * out of word zero, and for those that remain, the "OR" needs to occur
 265          * at the caller side. This debug check helps to catch cases where the
 266          * caller wants to OR but has forgotten to do so.
 267          */
 268         QBMAN_BUG_ON((*v & cmd_verb) != *v);
 269         *v = cmd_verb | p->mc.valid_bit;
 270         qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
 271 #ifdef QBMAN_CHECKING
 272         p->mc.check = swp_mc_can_poll;
 273 #endif
 274 }
 275
 276 void *qbman_swp_mc_result(struct qbman_swp *p)
 277 {
 278         uint32_t *ret, verb;
 279 #ifdef QBMAN_CHECKING
 280         QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll);
 281 #endif
 282         qbman_cena_invalidate_prefetch(&p->sys,
 283                                        QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 284         ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 285         /* Remove the valid-bit - command completed if the rest is non-zero */
 286         verb = ret[0] & ~QB_VALID_BIT;
 287         if (!verb)
 288                 return NULL;
 289 #ifdef QBMAN_CHECKING
 290         p->mc.check = swp_mc_can_start;
 291 #endif
 292         p->mc.valid_bit ^= QB_VALID_BIT;
 293         return ret;
 294 }
 295
 296 /***********/
 297 /* Enqueue */
 298 /***********/
 299
 300 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
 301 enum qb_enqueue_commands {
 302         enqueue_empty = 0,
 303         enqueue_response_always = 1,
 304         enqueue_rejects_to_fq = 2
 305 };
 306
 307 #define QB_ENQUEUE_CMD_EC_OPTION_MASK        0x3
 308 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
 309 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
 310 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
 311 #define QB_ENQUEUE_CMD_DCA_PK_SHIFT          6
 312 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
 313 #define QB_ENQUEUE_CMD_NLIS_SHIFT            14
 314 #define QB_ENQUEUE_CMD_IS_NESN_SHIFT         15
 315
 316 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
 317 {
 318         memset(d, 0, sizeof(*d));
 319 }
 320
 321 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
 322 {
 323         d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
 324         if (respond_success)
 325                 d->eq.verb |= enqueue_response_always;
 326         else
 327                 d->eq.verb |= enqueue_rejects_to_fq;
 328 }
 329
 330 void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success,
 331                            uint16_t opr_id, uint16_t seqnum, int incomplete)
 332 {
 333         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 334         if (respond_success)
 335                 d->eq.verb |= enqueue_response_always;
 336         else
 337                 d->eq.verb |= enqueue_rejects_to_fq;
 338
 339         d->eq.orpid = opr_id;
 340         d->eq.seqnum = seqnum;
 341         if (incomplete)
 342                 d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT;
 343         else
 344                 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 345 }
 346
 347 void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id,
 348                                 uint16_t seqnum)
 349 {
 350         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 351         d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
 352         d->eq.orpid = opr_id;
 353         d->eq.seqnum = seqnum;
 354         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 355         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT);
 356 }
 357
 358 void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id,
 359                                 uint16_t seqnum)
 360 {
 361         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 362         d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
 363         d->eq.orpid = opr_id;
 364         d->eq.seqnum = seqnum;
 365         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 366         d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT;
 367 }
 368
 369 void qbman_eq_desc_set_response(struct qbman_eq_desc *d,
 370                                 dma_addr_t storage_phys,
 371                                 int stash)
 372 {
 373         d->eq.rsp_addr = storage_phys;
 374         d->eq.wae = stash;
 375 }
 376
 377 void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token)
 378 {
 379         d->eq.rspid = token;
 380 }
 381
 382 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid)
 383 {
 384         d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
 385         d->eq.tgtid = fqid;
 386 }
 387
 388 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid,
 389                           uint16_t qd_bin, uint8_t qd_prio)
 390 {
 391         d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
 392         d->eq.tgtid = qdid;
 393         d->eq.qdbin = qd_bin;
 394         d->eq.qpri = qd_prio;
 395 }
 396
 397 void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable)
 398 {
 399         if (enable)
 400                 d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT;
 401         else
 402                 d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT);
 403 }
 404
 405 void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable,
 406                            uint8_t dqrr_idx, int park)
 407 {
 408         if (enable) {
 409                 d->eq.dca = dqrr_idx;
 410                 if (park)
 411                         d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT;
 412                 else
 413                         d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT);
 414                 d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT;
 415         } else {
 416                 d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT);
 417         }
 418 }
 419
 420 #define EQAR_IDX(eqar)     ((eqar) & 0x7)
 421 #define EQAR_VB(eqar)      ((eqar) & 0x80)
 422 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
 423
 424 static int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
 425                                         const struct qbman_eq_desc *d,
 426                                         const struct qbman_fd *fd)
 427 {
 428         uint32_t *p;
 429         const uint32_t *cl = qb_cl(d);
 430         uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR);
 431
 432         pr_debug("EQAR=%08x\n", eqar);
 433         if (!EQAR_SUCCESS(eqar))
 434                 return -EBUSY;
 435         p = qbman_cena_write_start_wo_shadow(&s->sys,
 436                                         QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 437         memcpy(&p[1], &cl[1], 28);
 438         memcpy(&p[8], fd, sizeof(*fd));
 439         /* Set the verb byte, have to substitute in the valid-bit */
 440         lwsync();
 441         p[0] = cl[0] | EQAR_VB(eqar);
 442         qbman_cena_write_complete_wo_shadow(&s->sys,
 443                                         QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 444         return 0;
 445 }
 446
 447 static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
 448                                        const struct qbman_eq_desc *d,
 449                                        const struct qbman_fd *fd)
 450 {
 451         uint32_t *p;
 452         const uint32_t *cl = qb_cl(d);
 453         uint32_t eqcr_ci;
 454         uint8_t diff;
 455
 456         if (!s->eqcr.available) {
 457                 eqcr_ci = s->eqcr.ci;
 458                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 459                                 QBMAN_CENA_SWP_EQCR_CI) & 0xF;
 460                 diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
 461                                    eqcr_ci, s->eqcr.ci);
 462                 s->eqcr.available += diff;
 463                 if (!diff)
 464                         return -EBUSY;
 465         }
 466
 467         p = qbman_cena_write_start_wo_shadow(&s->sys,
 468                                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
 469         memcpy(&p[1], &cl[1], 28);
 470         memcpy(&p[8], fd, sizeof(*fd));
 471         lwsync();
 472
 473         /* Set the verb byte, have to substitute in the valid-bit */
 474         p[0] = cl[0] | s->eqcr.pi_vb;
 475         qbman_cena_write_complete_wo_shadow(&s->sys,
 476                                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & 7));
 477         s->eqcr.pi++;
 478         s->eqcr.pi &= 0xF;
 479         s->eqcr.available--;
 480         if (!(s->eqcr.pi & 7))
 481                 s->eqcr.pi_vb ^= QB_VALID_BIT;
 482
 483         return 0;
 484 }
 485
 486 int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
 487                       const struct qbman_fd *fd)
 488 {
 489         if (s->sys.eqcr_mode == qman_eqcr_vb_array)
 490                 return qbman_swp_enqueue_array_mode(s, d, fd);
 491         else    /* Use ring mode by default */
 492                 return qbman_swp_enqueue_ring_mode(s, d, fd);
 493 }
 494
 495 int qbman_swp_enqueue_multiple(struct qbman_swp *s,
 496                                const struct qbman_eq_desc *d,
 497                                const struct qbman_fd *fd,
 498                                uint32_t *flags,
 499                                int num_frames)
 500 {
 501         uint32_t *p;
 502         const uint32_t *cl = qb_cl(d);
 503         uint32_t eqcr_ci, eqcr_pi;
 504         uint8_t diff;
 505         int i, num_enqueued = 0;
 506         uint64_t addr_cena;
 507
 508         if (!s->eqcr.available) {
 509                 eqcr_ci = s->eqcr.ci;
 510                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 511                                 QBMAN_CENA_SWP_EQCR_CI) & 0xF;
 512                 diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
 513                                    eqcr_ci, s->eqcr.ci);
 514                 s->eqcr.available += diff;
 515                 if (!diff)
 516                         return 0;
 517         }
 518
 519         eqcr_pi = s->eqcr.pi;
 520         num_enqueued = (s->eqcr.available < num_frames) ?
 521                         s->eqcr.available : num_frames;
 522         s->eqcr.available -= num_enqueued;
 523         /* Fill in the EQCR ring */
 524         for (i = 0; i < num_enqueued; i++) {
 525                 p = qbman_cena_write_start_wo_shadow(&s->sys,
 526                                         QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
 527                 memcpy(&p[1], &cl[1], 28);
 528                 memcpy(&p[8], &fd[i], sizeof(*fd));
 529                 eqcr_pi++;
 530                 eqcr_pi &= 0xF;
 531         }
 532
 533         lwsync();
 534
 535         /* Set the verb byte, have to substitute in the valid-bit */
 536         eqcr_pi = s->eqcr.pi;
 537         for (i = 0; i < num_enqueued; i++) {
 538                 p = qbman_cena_write_start_wo_shadow(&s->sys,
 539                                         QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
 540                 p[0] = cl[0] | s->eqcr.pi_vb;
 541                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
 542                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
 543
 544                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
 545                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
 546                 }
 547                 eqcr_pi++;
 548                 eqcr_pi &= 0xF;
 549                 if (!(eqcr_pi & 7))
 550                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 551         }
 552
 553         /* Flush all the cacheline without load/store in between */
 554         eqcr_pi = s->eqcr.pi;
 555         addr_cena = (size_t)s->sys.addr_cena;
 556         for (i = 0; i < num_enqueued; i++) {
 557                 dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
 558                 eqcr_pi++;
 559                 eqcr_pi &= 0xF;
 560         }
 561         s->eqcr.pi = eqcr_pi;
 562
 563         return num_enqueued;
 564 }
 565
 566 int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
 567                                     const struct qbman_eq_desc *d,
 568                                     const struct qbman_fd *fd,
 569                                     int num_frames)
 570 {
 571         uint32_t *p;
 572         const uint32_t *cl;
 573         uint32_t eqcr_ci, eqcr_pi;
 574         uint8_t diff;
 575         int i, num_enqueued = 0;
 576         uint64_t addr_cena;
 577
 578         if (!s->eqcr.available) {
 579                 eqcr_ci = s->eqcr.ci;
 580                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 581                                 QBMAN_CENA_SWP_EQCR_CI) & 0xF;
 582                 diff = qm_cyc_diff(QBMAN_EQCR_SIZE,
 583                                    eqcr_ci, s->eqcr.ci);
 584                 s->eqcr.available += diff;
 585                 if (!diff)
 586                         return 0;
 587         }
 588
 589         eqcr_pi = s->eqcr.pi;
 590         num_enqueued = (s->eqcr.available < num_frames) ?
 591                         s->eqcr.available : num_frames;
 592         s->eqcr.available -= num_enqueued;
 593         /* Fill in the EQCR ring */
 594         for (i = 0; i < num_enqueued; i++) {
 595                 p = qbman_cena_write_start_wo_shadow(&s->sys,
 596                                         QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
 597                 cl = qb_cl(&d[i]);
 598                 memcpy(&p[1], &cl[1], 28);
 599                 memcpy(&p[8], &fd[i], sizeof(*fd));
 600                 eqcr_pi++;
 601                 eqcr_pi &= 0xF;
 602         }
 603
 604         lwsync();
 605
 606         /* Set the verb byte, have to substitute in the valid-bit */
 607         eqcr_pi = s->eqcr.pi;
 608         for (i = 0; i < num_enqueued; i++) {
 609                 p = qbman_cena_write_start_wo_shadow(&s->sys,
 610                                         QBMAN_CENA_SWP_EQCR(eqcr_pi & 7));
 611                 cl = qb_cl(&d[i]);
 612                 p[0] = cl[0] | s->eqcr.pi_vb;
 613                 eqcr_pi++;
 614                 eqcr_pi &= 0xF;
 615                 if (!(eqcr_pi & 7))
 616                         s->eqcr.pi_vb ^= QB_VALID_BIT;
 617         }
 618
 619         /* Flush all the cacheline without load/store in between */
 620         eqcr_pi = s->eqcr.pi;
 621         addr_cena = (size_t)s->sys.addr_cena;
 622         for (i = 0; i < num_enqueued; i++) {
 623                 dcbf((addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & 7)));
 624                 eqcr_pi++;
 625                 eqcr_pi &= 0xF;
 626         }
 627         s->eqcr.pi = eqcr_pi;
 628
 629         return num_enqueued;
 630 }
 631
 632 /*************************/
 633 /* Static (push) dequeue */
 634 /*************************/
 635
 636 void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled)
 637 {
 638         uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
 639
 640         QBMAN_BUG_ON(channel_idx > 15);
 641         *enabled = src | (1 << channel_idx);
 642 }
 643
 644 void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable)
 645 {
 646         uint16_t dqsrc;
 647
 648         QBMAN_BUG_ON(channel_idx > 15);
 649         if (enable)
 650                 s->sdq |= 1 << channel_idx;
 651         else
 652                 s->sdq &= ~(1 << channel_idx);
 653
 654         /* Read make the complete src map.  If no channels are enabled
 655          * the SDQCR must be 0 or else QMan will assert errors
 656          */
 657         dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
 658         if (dqsrc != 0)
 659                 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq);
 660         else
 661                 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, 0);
 662 }
 663
 664 /***************************/
 665 /* Volatile (pull) dequeue */
 666 /***************************/
 667
 668 /* These should be const, eventually */
 669 #define QB_VDQCR_VERB_DCT_SHIFT    0
 670 #define QB_VDQCR_VERB_DT_SHIFT     2
 671 #define QB_VDQCR_VERB_RLS_SHIFT    4
 672 #define QB_VDQCR_VERB_WAE_SHIFT    5
 673
 674 enum qb_pull_dt_e {
 675         qb_pull_dt_channel,
 676         qb_pull_dt_workqueue,
 677         qb_pull_dt_framequeue
 678 };
 679
 680 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
 681 {
 682         memset(d, 0, sizeof(*d));
 683 }
 684
 685 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
 686                                  struct qbman_result *storage,
 687                                  dma_addr_t storage_phys,
 688                                  int stash)
 689 {
 690         d->pull.rsp_addr_virt = (size_t)storage;
 691
 692         if (!storage) {
 693                 d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
 694                 return;
 695         }
 696         d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
 697         if (stash)
 698                 d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
 699         else
 700                 d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
 701
 702         d->pull.rsp_addr = storage_phys;
 703 }
 704
 705 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, uint8_t numframes)
 706 {
 707         d->pull.numf = numframes - 1;
 708 }
 709
 710 void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token)
 711 {
 712         d->pull.tok = token;
 713 }
 714
 715 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid)
 716 {
 717         d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
 718         d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
 719         d->pull.dq_src = fqid;
 720 }
 721
 722 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid,
 723                             enum qbman_pull_type_e dct)
 724 {
 725         d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
 726         d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
 727         d->pull.dq_src = wqid;
 728 }
 729
 730 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid,
 731                                  enum qbman_pull_type_e dct)
 732 {
 733         d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
 734         d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
 735         d->pull.dq_src = chid;
 736 }
 737
 738 int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
 739 {
 740         uint32_t *p;
 741         uint32_t *cl = qb_cl(d);
 742
 743         if (!atomic_dec_and_test(&s->vdq.busy)) {
 744                 atomic_inc(&s->vdq.busy);
 745                 return -EBUSY;
 746         }
 747
 748         d->pull.tok = s->sys.idx + 1;
 749         s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
 750         p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
 751         memcpy(&p[1], &cl[1], 12);
 752
 753         /* Set the verb byte, have to substitute in the valid-bit */
 754         lwsync();
 755         p[0] = cl[0] | s->vdq.valid_bit;
 756         s->vdq.valid_bit ^= QB_VALID_BIT;
 757         qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
 758
 759         return 0;
 760 }
 761
 762 /****************/
 763 /* Polling DQRR */
 764 /****************/
 765
 766 #define QMAN_DQRR_PI_MASK              0xf
 767
 768 #define QBMAN_RESULT_DQ        0x60
 769 #define QBMAN_RESULT_FQRN      0x21
 770 #define QBMAN_RESULT_FQRNI     0x22
 771 #define QBMAN_RESULT_FQPN      0x24
 772 #define QBMAN_RESULT_FQDAN     0x25
 773 #define QBMAN_RESULT_CDAN      0x26
 774 #define QBMAN_RESULT_CSCN_MEM  0x27
 775 #define QBMAN_RESULT_CGCU      0x28
 776 #define QBMAN_RESULT_BPSCN     0x29
 777 #define QBMAN_RESULT_CSCN_WQ   0x2a
 778
 779 #include <rte_prefetch.h>
 780
 781 void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s)
 782 {
 783         const struct qbman_result *p;
 784
 785         p = qbman_cena_read_wo_shadow(&s->sys,
 786                 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
 787         rte_prefetch0(p);
 788 }
 789
 790 /* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry
 791  * only once, so repeated calls can return a sequence of DQRR entries, without
 792  * requiring they be consumed immediately or in any particular order.
 793  */
 794 const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
 795 {
 796         uint32_t verb;
 797         uint32_t response_verb;
 798         uint32_t flags;
 799         const struct qbman_result *p;
 800
 801         /* Before using valid-bit to detect if something is there, we have to
 802          * handle the case of the DQRR reset bug...
 803          */
 804         if (unlikely(s->dqrr.reset_bug)) {
 805                 /* We pick up new entries by cache-inhibited producer index,
 806                  * which means that a non-coherent mapping would require us to
 807                  * invalidate and read *only* once that PI has indicated that
 808                  * there's an entry here. The first trip around the DQRR ring
 809                  * will be much less efficient than all subsequent trips around
 810                  * it...
 811                  */
 812                 uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) &
 813                              QMAN_DQRR_PI_MASK;
 814
 815                 /* there are new entries if pi != next_idx */
 816                 if (pi == s->dqrr.next_idx)
 817                         return NULL;
 818
 819                 /* if next_idx is/was the last ring index, and 'pi' is
 820                  * different, we can disable the workaround as all the ring
 821                  * entries have now been DMA'd to so valid-bit checking is
 822                  * repaired. Note: this logic needs to be based on next_idx
 823                  * (which increments one at a time), rather than on pi (which
 824                  * can burst and wrap-around between our snapshots of it).
 825                  */
 826                 QBMAN_BUG_ON((s->dqrr.dqrr_size - 1) < 0);
 827                 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1u)) {
 828                         pr_debug("DEBUG: next_idx=%d, pi=%d, clear reset bug\n",
 829                                  s->dqrr.next_idx, pi);
 830                         s->dqrr.reset_bug = 0;
 831                 }
 832                 qbman_cena_invalidate_prefetch(&s->sys,
 833                                         QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
 834         }
 835         p = qbman_cena_read_wo_shadow(&s->sys,
 836                                       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
 837         verb = p->dq.verb;
 838
 839         /* If the valid-bit isn't of the expected polarity, nothing there. Note,
 840          * in the DQRR reset bug workaround, we shouldn't need to skip these
 841          * check, because we've already determined that a new entry is available
 842          * and we've invalidated the cacheline before reading it, so the
 843          * valid-bit behaviour is repaired and should tell us what we already
 844          * knew from reading PI.
 845          */
 846         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
 847                 return NULL;
 848
 849         /* There's something there. Move "next_idx" attention to the next ring
 850          * entry (and prefetch it) before returning what we found.
 851          */
 852         s->dqrr.next_idx++;
 853         if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
 854                 s->dqrr.next_idx = 0;
 855                 s->dqrr.valid_bit ^= QB_VALID_BIT;
 856         }
 857         /* If this is the final response to a volatile dequeue command
 858          * indicate that the vdq is no longer busy
 859          */
 860         flags = p->dq.stat;
 861         response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
 862         if ((response_verb == QBMAN_RESULT_DQ) &&
 863             (flags & QBMAN_DQ_STAT_VOLATILE) &&
 864             (flags & QBMAN_DQ_STAT_EXPIRED))
 865                 atomic_inc(&s->vdq.busy);
 866
 867         return p;
 868 }
 869
 870 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
 871 void qbman_swp_dqrr_consume(struct qbman_swp *s,
 872                             const struct qbman_result *dq)
 873 {
 874         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
 875 }
 876
 877 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
 878 void qbman_swp_dqrr_idx_consume(struct qbman_swp *s,
 879                             uint8_t dqrr_index)
 880 {
 881         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, dqrr_index);
 882 }
 883
 884 /*********************************/
 885 /* Polling user-provided storage */
 886 /*********************************/
 887 int qbman_result_has_new_result(struct qbman_swp *s,
 888                                 struct qbman_result *dq)
 889 {
 890         if (dq->dq.tok == 0)
 891                 return 0;
 892
 893         /*
 894          * Set token to be 0 so we will detect change back to 1
 895          * next time the looping is traversed. Const is cast away here
 896          * as we want users to treat the dequeue responses as read only.
 897          */
 898         ((struct qbman_result *)dq)->dq.tok = 0;
 899
 900         /*
 901          * VDQCR "no longer busy" hook - not quite the same as DQRR, because the
 902          * fact "VDQCR" shows busy doesn't mean that we hold the result that
 903          * makes it available. Eg. we may be looking at our 10th dequeue result,
 904          * having released VDQCR after the 1st result and it is now busy due to
 905          * some other command!
 906          */
 907         if (s->vdq.storage == dq) {
 908                 s->vdq.storage = NULL;
 909                 atomic_inc(&s->vdq.busy);
 910         }
 911
 912         return 1;
 913 }
 914
 915 int qbman_check_new_result(struct qbman_result *dq)
 916 {
 917         if (dq->dq.tok == 0)
 918                 return 0;
 919
 920         /*
 921          * Set token to be 0 so we will detect change back to 1
 922          * next time the looping is traversed. Const is cast away here
 923          * as we want users to treat the dequeue responses as read only.
 924          */
 925         ((struct qbman_result *)dq)->dq.tok = 0;
 926
 927         return 1;
 928 }
 929
 930 int qbman_check_command_complete(struct qbman_result *dq)
 931 {
 932         struct qbman_swp *s;
 933
 934         if (dq->dq.tok == 0)
 935                 return 0;
 936
 937         s = portal_idx_map[dq->dq.tok - 1];
 938         /*
 939          * VDQCR "no longer busy" hook - not quite the same as DQRR, because the
 940          * fact "VDQCR" shows busy doesn't mean that we hold the result that
 941          * makes it available. Eg. we may be looking at our 10th dequeue result,
 942          * having released VDQCR after the 1st result and it is now busy due to
 943          * some other command!
 944          */
 945         if (s->vdq.storage == dq) {
 946                 s->vdq.storage = NULL;
 947                 atomic_inc(&s->vdq.busy);
 948         }
 949
 950         return 1;
 951 }
 952
 953 /********************************/
 954 /* Categorising qbman results   */
 955 /********************************/
 956
 957 static inline int __qbman_result_is_x(const struct qbman_result *dq,
 958                                       uint8_t x)
 959 {
 960         uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK;
 961
 962         return (response_verb == x);
 963 }
 964
 965 int qbman_result_is_DQ(const struct qbman_result *dq)
 966 {
 967         return __qbman_result_is_x(dq, QBMAN_RESULT_DQ);
 968 }
 969
 970 int qbman_result_is_FQDAN(const struct qbman_result *dq)
 971 {
 972         return __qbman_result_is_x(dq, QBMAN_RESULT_FQDAN);
 973 }
 974
 975 int qbman_result_is_CDAN(const struct qbman_result *dq)
 976 {
 977         return __qbman_result_is_x(dq, QBMAN_RESULT_CDAN);
 978 }
 979
 980 int qbman_result_is_CSCN(const struct qbman_result *dq)
 981 {
 982         return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) ||
 983                 __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ);
 984 }
 985
 986 int qbman_result_is_BPSCN(const struct qbman_result *dq)
 987 {
 988         return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN);
 989 }
 990
 991 int qbman_result_is_CGCU(const struct qbman_result *dq)
 992 {
 993         return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU);
 994 }
 995
 996 int qbman_result_is_FQRN(const struct qbman_result *dq)
 997 {
 998         return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN);
 999 }
1000
1001 int qbman_result_is_FQRNI(const struct qbman_result *dq)
1002 {
1003         return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI);
1004 }
1005
1006 int qbman_result_is_FQPN(const struct qbman_result *dq)
1007 {
1008         return __qbman_result_is_x(dq, QBMAN_RESULT_FQPN);
1009 }
1010
1011 /*********************************/
1012 /* Parsing frame dequeue results */
1013 /*********************************/
1014
1015 /* These APIs assume qbman_result_is_DQ() is TRUE */
1016
1017 uint8_t qbman_result_DQ_flags(const struct qbman_result *dq)
1018 {
1019         return dq->dq.stat;
1020 }
1021
1022 uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq)
1023 {
1024         return dq->dq.seqnum;
1025 }
1026
1027 uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq)
1028 {
1029         return dq->dq.oprid;
1030 }
1031
1032 uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq)
1033 {
1034         return dq->dq.fqid;
1035 }
1036
1037 uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq)
1038 {
1039         return dq->dq.fq_byte_cnt;
1040 }
1041
1042 uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq)
1043 {
1044         return dq->dq.fq_frm_cnt;
1045 }
1046
1047 uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq)
1048 {
1049         return dq->dq.fqd_ctx;
1050 }
1051
1052 const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq)
1053 {
1054         return (const struct qbman_fd *)&dq->dq.fd[0];
1055 }
1056
1057 /**************************************/
1058 /* Parsing state-change notifications */
1059 /**************************************/
1060 uint8_t qbman_result_SCN_state(const struct qbman_result *scn)
1061 {
1062         return scn->scn.state;
1063 }
1064
1065 uint32_t qbman_result_SCN_rid(const struct qbman_result *scn)
1066 {
1067         return scn->scn.rid_tok;
1068 }
1069
1070 uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn)
1071 {
1072         return scn->scn.ctx;
1073 }
1074
1075 /*****************/
1076 /* Parsing BPSCN */
1077 /*****************/
1078 uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn)
1079 {
1080         return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF;
1081 }
1082
1083 int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn)
1084 {
1085         return !(int)(qbman_result_SCN_state(scn) & 0x1);
1086 }
1087
1088 int qbman_result_bpscn_is_depleted(const struct qbman_result *scn)
1089 {
1090         return (int)(qbman_result_SCN_state(scn) & 0x2);
1091 }
1092
1093 int qbman_result_bpscn_is_surplus(const struct qbman_result *scn)
1094 {
1095         return (int)(qbman_result_SCN_state(scn) & 0x4);
1096 }
1097
1098 uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn)
1099 {
1100         return qbman_result_SCN_ctx(scn);
1101 }
1102
1103 /*****************/
1104 /* Parsing CGCU  */
1105 /*****************/
1106 uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn)
1107 {
1108         return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF;
1109 }
1110
1111 uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn)
1112 {
1113         return qbman_result_SCN_ctx(scn);
1114 }
1115
1116 /******************/
1117 /* Buffer release */
1118 /******************/
1119 #define QB_BR_RC_VALID_SHIFT  5
1120 #define QB_BR_RCDI_SHIFT      6
1121
1122 void qbman_release_desc_clear(struct qbman_release_desc *d)
1123 {
1124         memset(d, 0, sizeof(*d));
1125         d->br.verb = 1 << QB_BR_RC_VALID_SHIFT;
1126 }
1127
1128 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid)
1129 {
1130         d->br.bpid = bpid;
1131 }
1132
1133 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
1134 {
1135         if (enable)
1136                 d->br.verb |= 1 << QB_BR_RCDI_SHIFT;
1137         else
1138                 d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT);
1139 }
1140
1141 #define RAR_IDX(rar)     ((rar) & 0x7)
1142 #define RAR_VB(rar)      ((rar) & 0x80)
1143 #define RAR_SUCCESS(rar) ((rar) & 0x100)
1144
1145 int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
1146                       const uint64_t *buffers, unsigned int num_buffers)
1147 {
1148         uint32_t *p;
1149         const uint32_t *cl = qb_cl(d);
1150         uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
1151
1152         pr_debug("RAR=%08x\n", rar);
1153         if (!RAR_SUCCESS(rar))
1154                 return -EBUSY;
1155
1156         QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
1157
1158         /* Start the release command */
1159         p = qbman_cena_write_start_wo_shadow(&s->sys,
1160                                              QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
1161
1162         /* Copy the caller's buffer pointers to the command */
1163         u64_to_le32_copy(&p[2], buffers, num_buffers);
1164
1165         /* Set the verb byte, have to substitute in the valid-bit and the number
1166          * of buffers.
1167          */
1168         lwsync();
1169         p[0] = cl[0] | RAR_VB(rar) | num_buffers;
1170         qbman_cena_write_complete_wo_shadow(&s->sys,
1171                                             QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
1172
1173         return 0;
1174 }
1175
1176 /*******************/
1177 /* Buffer acquires */
1178 /*******************/
1179 struct qbman_acquire_desc {
1180         uint8_t verb;
1181         uint8_t reserved;
1182         uint16_t bpid;
1183         uint8_t num;
1184         uint8_t reserved2[59];
1185 };
1186
1187 struct qbman_acquire_rslt {
1188         uint8_t verb;
1189         uint8_t rslt;
1190         uint16_t reserved;
1191         uint8_t num;
1192         uint8_t reserved2[3];
1193         uint64_t buf[7];
1194 };
1195
1196 int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers,
1197                       unsigned int num_buffers)
1198 {
1199         struct qbman_acquire_desc *p;
1200         struct qbman_acquire_rslt *r;
1201
1202         if (!num_buffers || (num_buffers > 7))
1203                 return -EINVAL;
1204
1205         /* Start the management command */
1206         p = qbman_swp_mc_start(s);
1207
1208         if (!p)
1209                 return -EBUSY;
1210
1211         /* Encode the caller-provided attributes */
1212         p->bpid = bpid;
1213         p->num = num_buffers;
1214
1215         /* Complete the management command */
1216         r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
1217         if (unlikely(!r)) {
1218                 pr_err("qbman: acquire from BPID %d failed, no response\n",
1219                        bpid);
1220                 return -EIO;
1221         }
1222
1223         /* Decode the outcome */
1224         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE);
1225
1226         /* Determine success or failure */
1227         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1228                 pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n",
1229                        bpid, r->rslt);
1230                 return -EIO;
1231         }
1232
1233         QBMAN_BUG_ON(r->num > num_buffers);
1234
1235         /* Copy the acquired buffers to the caller's array */
1236         u64_from_le32_copy(buffers, &r->buf[0], r->num);
1237
1238         return (int)r->num;
1239 }
1240
1241 /*****************/
1242 /* FQ management */
1243 /*****************/
1244 struct qbman_alt_fq_state_desc {
1245         uint8_t verb;
1246         uint8_t reserved[3];
1247         uint32_t fqid;
1248         uint8_t reserved2[56];
1249 };
1250
1251 struct qbman_alt_fq_state_rslt {
1252         uint8_t verb;
1253         uint8_t rslt;
1254         uint8_t reserved[62];
1255 };
1256
1257 #define ALT_FQ_FQID_MASK 0x00FFFFFF
1258
1259 static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid,
1260                                   uint8_t alt_fq_verb)
1261 {
1262         struct qbman_alt_fq_state_desc *p;
1263         struct qbman_alt_fq_state_rslt *r;
1264
1265         /* Start the management command */
1266         p = qbman_swp_mc_start(s);
1267         if (!p)
1268                 return -EBUSY;
1269
1270         p->fqid = fqid & ALT_FQ_FQID_MASK;
1271
1272         /* Complete the management command */
1273         r = qbman_swp_mc_complete(s, p, alt_fq_verb);
1274         if (unlikely(!r)) {
1275                 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
1276                        alt_fq_verb);
1277                 return -EIO;
1278         }
1279
1280         /* Decode the outcome */
1281         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb);
1282
1283         /* Determine success or failure */
1284         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1285                 pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n",
1286                        fqid, alt_fq_verb, r->rslt);
1287                 return -EIO;
1288         }
1289
1290         return 0;
1291 }
1292
1293 int qbman_swp_fq_schedule(struct qbman_swp *s, uint32_t fqid)
1294 {
1295         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_SCHEDULE);
1296 }
1297
1298 int qbman_swp_fq_force(struct qbman_swp *s, uint32_t fqid)
1299 {
1300         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_FORCE);
1301 }
1302
1303 int qbman_swp_fq_xon(struct qbman_swp *s, uint32_t fqid)
1304 {
1305         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XON);
1306 }
1307
1308 int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid)
1309 {
1310         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XOFF);
1311 }
1312
1313 /**********************/
1314 /* Channel management */
1315 /**********************/
1316
1317 struct qbman_cdan_ctrl_desc {
1318         uint8_t verb;
1319         uint8_t reserved;
1320         uint16_t ch;
1321         uint8_t we;
1322         uint8_t ctrl;
1323         uint16_t reserved2;
1324         uint64_t cdan_ctx;
1325         uint8_t reserved3[48];
1326
1327 };
1328
1329 struct qbman_cdan_ctrl_rslt {
1330         uint8_t verb;
1331         uint8_t rslt;
1332         uint16_t ch;
1333         uint8_t reserved[60];
1334 };
1335
1336 /* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it
1337  * would be irresponsible to expose it.
1338  */
1339 #define CODE_CDAN_WE_EN    0x1
1340 #define CODE_CDAN_WE_CTX   0x4
1341
1342 static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid,
1343                               uint8_t we_mask, uint8_t cdan_en,
1344                               uint64_t ctx)
1345 {
1346         struct qbman_cdan_ctrl_desc *p;
1347         struct qbman_cdan_ctrl_rslt *r;
1348
1349         /* Start the management command */
1350         p = qbman_swp_mc_start(s);
1351         if (!p)
1352                 return -EBUSY;
1353
1354         /* Encode the caller-provided attributes */
1355         p->ch = channelid;
1356         p->we = we_mask;
1357         if (cdan_en)
1358                 p->ctrl = 1;
1359         else
1360                 p->ctrl = 0;
1361         p->cdan_ctx = ctx;
1362
1363         /* Complete the management command */
1364         r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
1365         if (unlikely(!r)) {
1366                 pr_err("qbman: wqchan config failed, no response\n");
1367                 return -EIO;
1368         }
1369
1370         /* Decode the outcome */
1371         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK)
1372                      != QBMAN_WQCHAN_CONFIGURE);
1373
1374         /* Determine success or failure */
1375         if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1376                 pr_err("CDAN cQID %d failed: code = 0x%02x\n",
1377                        channelid, r->rslt);
1378                 return -EIO;
1379         }
1380
1381         return 0;
1382 }
1383
1384 int qbman_swp_CDAN_set_context(struct qbman_swp *s, uint16_t channelid,
1385                                uint64_t ctx)
1386 {
1387         return qbman_swp_CDAN_set(s, channelid,
1388                                   CODE_CDAN_WE_CTX,
1389                                   0, ctx);
1390 }
1391
1392 int qbman_swp_CDAN_enable(struct qbman_swp *s, uint16_t channelid)
1393 {
1394         return qbman_swp_CDAN_set(s, channelid,
1395                                   CODE_CDAN_WE_EN,
1396                                   1, 0);
1397 }
1398
1399 int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid)
1400 {
1401         return qbman_swp_CDAN_set(s, channelid,
1402                                   CODE_CDAN_WE_EN,
1403                                   0, 0);
1404 }
1405
1406 int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid,
1407                                       uint64_t ctx)
1408 {
1409         return qbman_swp_CDAN_set(s, channelid,
1410                                   CODE_CDAN_WE_EN | CODE_CDAN_WE_CTX,
1411                                   1, ctx);
1412 }
1413
1414 uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr)
1415 {
1416         return QBMAN_IDX_FROM_DQRR(dqrr);
1417 }
1418
1419 struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx)
1420 {
1421         struct qbman_result *dq;
1422
1423         dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx));
1424         return dq;
1425 }