X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=drivers%2Fbus%2Fdpaa%2Fbase%2Fqbman%2Fqman.c;h=2b97671bf5b78420bbeee7716fd8d1840d6b18cc;hb=9aa8c7c855a82db086c866d7c1c5d49bfaffef0c;hp=87fec60d1a6028e09bc6fde2c66f72a409411ce2;hpb=169a9de21e263aa6599cdc2d87a45ae158d9f509;p=deb_dpdk.git

diff --git a/drivers/bus/dpaa/base/qbman/qman.c b/drivers/bus/dpaa/base/qbman/qman.c
index 87fec60d..2b97671b 100644
--- a/drivers/bus/dpaa/base/qbman/qman.c
+++ b/drivers/bus/dpaa/base/qbman/qman.c
@@ -1,45 +1,15 @@
-/*-
- * This file is provided under a dual BSD/GPLv2 license. When using or
- * redistributing this file, you may do so under either license.
- *
- *   BSD LICENSE
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
  *
  * Copyright 2008-2016 Freescale Semiconductor Inc.
- * Copyright 2017 NXP.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of the above-listed copyright holders nor the
- * names of any contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- *   GPL LICENSE SUMMARY
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
+ * Copyright 2017 NXP
  *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "qman.h"
 #include <rte_branch_prediction.h>
+#include <rte_dpaa_bus.h>
+#include <rte_eventdev.h>
+#include <rte_byteorder.h>
 
 /* Compilation constants */
 #define DQRR_MAXFILL	15
@@ -416,7 +386,9 @@ static inline void qm_eqcr_finish(struct qm_portal *portal)
 	qm_cl_invalidate(EQCR_CI);
 	eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
 
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 	DPAA_ASSERT(!eqcr->busy);
+#endif
 	if (pi != EQCR_PTR2IDX(eqcr->cursor))
 		pr_crit("losing uncommitted EQCR entries\n");
 	if (ci != eqcr->ci)
@@ -505,7 +477,9 @@ static inline void qm_mr_pvb_update(struct qm_portal *portal)
 	register struct qm_mr *mr = &portal->mr;
 	const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
 
+#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 	DPAA_ASSERT(mr->pmode == qm_mr_pvb);
+#endif
 	/* when accessing 'verb', use __raw_readb() to ensure that compiler
 	 * inlining doesn't try to optimise out "excess reads".
 	 */
@@ -532,7 +506,12 @@ struct qman_portal *qman_create_portal(
 
 	p = &portal->p;
 
-	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
+	if (dpaa_svr_family == SVR_LS1043A_FAMILY)
+		portal->use_eqcr_ci_stashing = 3;
+	else
+		portal->use_eqcr_ci_stashing =
+					((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
+
 	/*
 	 * prep the low-level portal struct with the mapped addresses from the
 	 * config, everything that follows depends on it and "config" is more
@@ -545,7 +524,7 @@ struct qman_portal *qman_create_portal(
 	 * and stash with high-than-DQRR priority.
 	 */
 	if (qm_eqcr_init(p, qm_eqcr_pvb,
-			 portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
+			 portal->use_eqcr_ci_stashing, 1)) {
 		pr_err("Qman EQCR initialisation failed\n");
 		goto fail_eqcr;
 	}
@@ -644,11 +623,50 @@ fail_eqcr:
 	return NULL;
 }
 
+#define MAX_GLOBAL_PORTALS 8
+static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
+rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
+
+static struct qman_portal *
+qman_alloc_global_portal(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
+		if (rte_atomic16_test_and_set(&global_portals_used[i]))
+			return &global_portals[i];
+	}
+	pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS);
+
+	return NULL;
+}
+
+static int
+qman_free_global_portal(struct qman_portal *portal)
+{
+	unsigned int i;
+
+	for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
+		if (&global_portals[i] == portal) {
+			rte_atomic16_clear(&global_portals_used[i]);
+			return 0;
+		}
+	}
+	return -1;
+}
+
 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
-					      const struct qman_cgrs *cgrs)
+					      const struct qman_cgrs *cgrs,
+					      int alloc)
 {
 	struct qman_portal *res;
-	struct qman_portal *portal = get_affine_portal();
+	struct qman_portal *portal;
+
+	if (alloc)
+		portal = qman_alloc_global_portal();
+	else
+		portal = get_affine_portal();
+
 	/* A criteria for calling this function (from qman_driver.c) is that
 	 * we're already affine to the cpu and won't schedule onto another cpu.
 	 */
@@ -698,13 +716,18 @@ void qman_destroy_portal(struct qman_portal *qm)
 	spin_lock_destroy(&qm->cgr_lock);
 }
 
-const struct qm_portal_config *qman_destroy_affine_portal(void)
+const struct qm_portal_config *
+qman_destroy_affine_portal(struct qman_portal *qp)
 {
 	/* We don't want to redirect if we're a slave, use "raw" */
-	struct qman_portal *qm = get_affine_portal();
+	struct qman_portal *qm;
 	const struct qm_portal_config *pcfg;
 	int cpu;
 
+	if (qp == NULL)
+		qm = get_affine_portal();
+	else
+		qm = qp;
 	pcfg = qm->config;
 	cpu = pcfg->cpu;
 
@@ -713,6 +736,9 @@ const struct qm_portal_config *qman_destroy_affine_portal(void)
 	spin_lock(&affine_mask_lock);
 	CPU_CLR(cpu, &affine_mask);
 	spin_unlock(&affine_mask_lock);
+
+	qman_free_global_portal(qm);
+
 	return pcfg;
 }
 
@@ -929,7 +955,7 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 	do {
 		qm_dqrr_pvb_update(&p->p);
 		dq = qm_dqrr_current(&p->p);
-		if (!dq)
+		if (unlikely(!dq))
 			break;
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 	/* If running on an LE system the fields of the
@@ -1025,6 +1051,137 @@ u16 qman_affine_channel(int cpu)
 	return affine_channels[cpu];
 }
 
+unsigned int qman_portal_poll_rx(unsigned int poll_limit,
+				 void **bufs,
+				 struct qman_portal *p)
+{
+	struct qm_portal *portal = &p->p;
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE];
+	struct qman_fq *fq[QM_DQRR_SIZE];
+	unsigned int limit = 0, rx_number = 0;
+	uint32_t consume = 0;
+
+	do {
+		qm_dqrr_pvb_update(&p->p);
+		if (!dqrr->fill)
+			break;
+
+		dq[rx_number] = dqrr->cursor;
+		dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1);
+		/* Prefetch the next DQRR entry */
+		rte_prefetch0(dqrr->cursor);
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+		/* If running on an LE system the fields of the
+		 * dequeue entry must be swapper.  Because the
+		 * QMan HW will ignore writes the DQRR entry is
+		 * copied and the index stored within the copy
+		 */
+		shadow[rx_number] =
+			&p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])];
+		shadow[rx_number]->fd.opaque_addr =
+			dq[rx_number]->fd.opaque_addr;
+		shadow[rx_number]->fd.addr =
+			be40_to_cpu(dq[rx_number]->fd.addr);
+		shadow[rx_number]->fd.opaque =
+			be32_to_cpu(dq[rx_number]->fd.opaque);
+#else
+		shadow = dq;
+#endif
+
+		/* SDQCR: context_b points to the FQ */
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+		fq[rx_number] = qman_fq_lookup_table[be32_to_cpu(
+						dq[rx_number]->contextB)];
+#else
+		fq[rx_number] = (void *)(uintptr_t)be32_to_cpu(dq->contextB);
+#endif
+		fq[rx_number]->cb.dqrr_prepare(shadow[rx_number],
+						 &bufs[rx_number]);
+
+		consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number])));
+		rx_number++;
+		--dqrr->fill;
+	} while (++limit < poll_limit);
+
+	if (rx_number)
+		fq[0]->cb.dqrr_dpdk_pull_cb(fq, shadow, bufs, rx_number);
+
+	/* Consume all the DQRR enries together */
+	qm_out(DQRR_DCAP, (1 << 8) | consume);
+
+	return rx_number;
+}
+
+u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
+			void **bufs)
+{
+	const struct qm_dqrr_entry *dq;
+	struct qman_fq *fq;
+	enum qman_cb_dqrr_result res;
+	unsigned int limit = 0;
+	struct qman_portal *p = get_affine_portal();
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+	struct qm_dqrr_entry *shadow;
+#endif
+	unsigned int rx_number = 0;
+
+	do {
+		qm_dqrr_pvb_update(&p->p);
+		dq = qm_dqrr_current(&p->p);
+		if (!dq)
+			break;
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+		/*
+		 * If running on an LE system the fields of the
+		 * dequeue entry must be swapper.  Because the
+		 * QMan HW will ignore writes the DQRR entry is
+		 * copied and the index stored within the copy
+		 */
+		shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
+		*shadow = *dq;
+		dq = shadow;
+		shadow->fqid = be32_to_cpu(shadow->fqid);
+		shadow->contextB = be32_to_cpu(shadow->contextB);
+		shadow->seqnum = be16_to_cpu(shadow->seqnum);
+		hw_fd_to_cpu(&shadow->fd);
+#endif
+
+	       /* SDQCR: context_b points to the FQ */
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+		fq = get_fq_table_entry(dq->contextB);
+#else
+		fq = (void *)(uintptr_t)dq->contextB;
+#endif
+		/* Now let the callback do its stuff */
+		res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq,
+					 dq, &bufs[rx_number]);
+		rx_number++;
+		/* Interpret 'dq' from a driver perspective. */
+		/*
+		 * Parking isn't possible unless HELDACTIVE was set. NB,
+		 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
+		 * check for HELDACTIVE to cover both.
+		 */
+		DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
+			    (res != qman_cb_dqrr_park));
+		if (res != qman_cb_dqrr_defer)
+			qm_dqrr_cdc_consume_1ptr(&p->p, dq,
+						 res == qman_cb_dqrr_park);
+		/* Move forward */
+		qm_dqrr_next(&p->p);
+		/*
+		 * Entry processed and consumed, increment our counter.  The
+		 * callback can request that we exit after consuming the
+		 * entry, and we also exit if we reach our processing limit,
+		 * so loop back only if neither of these conditions is met.
+		 */
+	} while (++limit < poll_limit);
+
+	return limit;
+}
+
 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
 {
 	struct qman_portal *p = get_affine_portal();
@@ -1119,37 +1276,44 @@ void qman_start_dequeues(void)
 		qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
 }
 
-void qman_static_dequeue_add(u32 pools)
+void qman_static_dequeue_add(u32 pools, struct qman_portal *qp)
 {
-	struct qman_portal *p = get_affine_portal();
+	struct qman_portal *p = qp ? qp : get_affine_portal();
 
 	pools &= p->config->pools;
 	p->sdqcr |= pools;
 	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
 }
 
-void qman_static_dequeue_del(u32 pools)
+void qman_static_dequeue_del(u32 pools, struct qman_portal *qp)
 {
-	struct qman_portal *p = get_affine_portal();
+	struct qman_portal *p = qp ? qp : get_affine_portal();
 
 	pools &= p->config->pools;
 	p->sdqcr &= ~pools;
 	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
 }
 
-u32 qman_static_dequeue_get(void)
+u32 qman_static_dequeue_get(struct qman_portal *qp)
 {
-	struct qman_portal *p = get_affine_portal();
+	struct qman_portal *p = qp ? qp : get_affine_portal();
 	return p->sdqcr;
 }
 
-void qman_dca(struct qm_dqrr_entry *dq, int park_request)
+void qman_dca(const struct qm_dqrr_entry *dq, int park_request)
 {
 	struct qman_portal *p = get_affine_portal();
 
 	qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
 }
 
+void qman_dca_index(u8 index, int park_request)
+{
+	struct qman_portal *p = get_affine_portal();
+
+	qm_dqrr_cdc_consume_1(&p->p, index, park_request);
+}
+
 /* Frame queue API */
 static const char *mcr_result_str(u8 result)
 {
@@ -1188,6 +1352,7 @@ int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
 	}
 	spin_lock_init(&fq->fqlock);
 	fq->fqid = fqid;
+	fq->fqid_le = cpu_to_be32(fqid);
 	fq->flags = flags;
 	fq->state = qman_fq_state_oos;
 	fq->cgr_groupid = 0;
@@ -1267,6 +1432,7 @@ void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
 	switch (fq->state) {
 	case qman_fq_state_parked:
 		DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
+		/* Fallthrough */
 	case qman_fq_state_oos:
 		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
 			qman_release_fqid(fq->fqid);
@@ -1694,6 +1860,28 @@ int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
 	return 0;
 }
 
+int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+
+	mcc = qm_mc_start(&p->p);
+	mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
+
+	if (mcr->result == QM_MCR_RESULT_OK)
+		*frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt);
+	else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
+		return -ERANGE;
+	else if (mcr->result != QM_MCR_RESULT_OK)
+		return -EIO;
+	return 0;
+}
+
 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
 {
 	struct qm_mc_command *mcc;
@@ -1974,7 +2162,7 @@ int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
 }
 
 int qman_enqueue_multi(struct qman_fq *fq,
-		       const struct qm_fd *fd,
+		       const struct qm_fd *fd, u32 *flags,
 		int frames_to_send)
 {
 	struct qman_portal *p = get_affine_portal();
@@ -1983,7 +2171,7 @@ int qman_enqueue_multi(struct qman_fq *fq,
 	register struct qm_eqcr *eqcr = &portal->eqcr;
 	struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
 
-	u8 i, diff, old_ci, sent = 0;
+	u8 i = 0, diff, old_ci, sent = 0;
 
 	/* Update the available entries if no entry is free */
 	if (!eqcr->available) {
@@ -1997,7 +2185,7 @@ int qman_enqueue_multi(struct qman_fq *fq,
 
 	/* try to send as many frames as possible */
 	while (eqcr->available && frames_to_send--) {
-		eq->fqid = cpu_to_be32(fq->fqid);
+		eq->fqid = fq->fqid_le;
 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 		eq->tag = cpu_to_be32(fq->key);
 #else
@@ -2007,6 +2195,76 @@ int qman_enqueue_multi(struct qman_fq *fq,
 		eq->fd.addr = cpu_to_be40(fd->addr);
 		eq->fd.status = cpu_to_be32(fd->status);
 		eq->fd.opaque = cpu_to_be32(fd->opaque);
+		if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) {
+			eq->dca = QM_EQCR_DCA_ENABLE |
+				((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK);
+		}
+		i++;
+		eq = (void *)((unsigned long)(eq + 1) &
+			(~(unsigned long)(QM_EQCR_SIZE << 6)));
+		eqcr->available--;
+		sent++;
+		fd++;
+	}
+	lwsync();
+
+	/* In order for flushes to complete faster, all lines are recorded in
+	 * 32 bit word.
+	 */
+	eq = eqcr->cursor;
+	for (i = 0; i < sent; i++) {
+		eq->__dont_write_directly__verb =
+			QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
+		prev_eq = eq;
+		eq = (void *)((unsigned long)(eq + 1) &
+			(~(unsigned long)(QM_EQCR_SIZE << 6)));
+		if (unlikely((prev_eq + 1) != eq))
+			eqcr->vbit ^= QM_EQCR_VERB_VBIT;
+	}
+
+	/* We need  to flush all the lines but without load/store operations
+	 * between them
+	 */
+	eq = eqcr->cursor;
+	for (i = 0; i < sent; i++) {
+		dcbf(eq);
+		eq = (void *)((unsigned long)(eq + 1) &
+			(~(unsigned long)(QM_EQCR_SIZE << 6)));
+	}
+	/* Update cursor for the next call */
+	eqcr->cursor = eq;
+	return sent;
+}
+
+int
+qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd,
+		      int frames_to_send)
+{
+	struct qman_portal *p = get_affine_portal();
+	struct qm_portal *portal = &p->p;
+
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
+
+	u8 i, diff, old_ci, sent = 0;
+
+	/* Update the available entries if no entry is free */
+	if (!eqcr->available) {
+		old_ci = eqcr->ci;
+		eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
+		diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+		eqcr->available += diff;
+		if (!diff)
+			return 0;
+	}
+
+	/* try to send as many frames as possible */
+	while (eqcr->available && frames_to_send--) {
+		eq->fqid = fq[sent]->fqid_le;
+		eq->fd.opaque_addr = fd->opaque_addr;
+		eq->fd.addr = cpu_to_be40(fd->addr);
+		eq->fd.status = cpu_to_be32(fd->status);
+		eq->fd.opaque = cpu_to_be32(fd->opaque);
 
 		eq = (void *)((unsigned long)(eq + 1) &
 			(~(unsigned long)(QM_EQCR_SIZE << 6)));