1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Netronome Systems, Inc.
8 * Authors: Vinayak Tammineedi <vinayak.tammineedi@netronome.com>
10 * Multiplexes the NFP BARs between NFP internal resources and
11 * implements the PCIe specific interface for generic CPP bus access.
13 * The BARs are managed and allocated if they are available.
14 * The generic CPP bus abstraction builds upon this BAR interface.
19 #if defined(RTE_BACKTRACE)
36 #include <rte_ethdev_pci.h>
37 #include <rte_string_fns.h>
40 #include "nfp_target.h"
41 #include "nfp6000/nfp6000.h"
43 #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0)
45 #define NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(_x) (((_x) & 0x1f) << 16)
46 #define NFP_PCIE_BAR_PCIE2CPP_BASEADDRESS(_x) (((_x) & 0xffff) << 0)
47 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT(_x) (((_x) & 0x3) << 27)
48 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT 0
49 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT 1
50 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE 3
51 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE(_x) (((_x) & 0x7) << 29)
52 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(_x) (((_x) >> 29) & 0x7)
53 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED 0
54 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK 1
55 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_TARGET 2
56 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL 3
57 #define NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(_x) (((_x) & 0xf) << 23)
58 #define NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(_x) (((_x) & 0x3) << 21)
61 * Minimal size of the PCIe cfg memory we depend on being mapped,
62 * queue controller and DMA controller don't have to be covered.
64 #define NFP_PCI_MIN_MAP_SIZE 0x080000
66 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize)
67 #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize)
68 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
69 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
70 #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4))
72 #define NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar, slot) \
73 (NFP_PCIE_BAR(0) + ((bar) * 8 + (slot)) * 4)
75 #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPBAR(bar, slot) \
76 (((bar) * 8 + (slot)) * 4)
79 * Define to enable a bit more verbose debug output.
80 * Set to 1 to enable a bit more verbose debug output.
83 struct nfp6000_area_priv;
86 * struct nfp_bar - describes BAR configuration and usage
87 * @nfp: backlink to owner
88 * @barcfg: cached contents of BAR config CSR
89 * @base: the BAR's base CPP offset
90 * @mask: mask for the BAR aperture (read only)
91 * @bitsize: bitsize of BAR aperture (read only)
92 * @index: index of the BAR
93 * @lock: lock to specify if bar is in use
94 * @refcnt: number of current users
95 * @iomem: mapped IO memory
99 struct nfp_pcie_user *nfp;
101 uint64_t base; /* CPP address base */
102 uint64_t mask; /* Bit mask of the bar */
103 uint32_t bitsize; /* Bit size of the bar */
112 struct nfp_pcie_user {
113 struct nfp_bar bar[NFP_BAR_MAX];
117 char busdev[BUSDEV_SZ];
123 nfp_bar_maptype(struct nfp_bar *bar)
125 return NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(bar->barcfg);
128 #define TARGET_WIDTH_32 4
129 #define TARGET_WIDTH_64 8
132 nfp_compute_bar(const struct nfp_bar *bar, uint32_t *bar_config,
133 uint64_t *bar_base, int tgt, int act, int tok,
134 uint64_t offset, size_t size, int width)
146 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
147 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT);
151 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
152 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT);
156 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
157 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE);
163 if (act != NFP_CPP_ACTION_RW && act != 0) {
164 /* Fixed CPP mapping with specific action */
165 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
168 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE
169 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED);
170 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(tgt);
171 newcfg |= NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(act);
172 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(tok);
174 if ((offset & mask) != ((offset + size - 1) & mask)) {
175 printf("BAR%d: Won't use for Fixed mapping\n",
177 printf("\t<%#llx,%#llx>, action=%d\n",
178 (unsigned long long)offset,
179 (unsigned long long)(offset + size), act);
180 printf("\tBAR too small (0x%llx).\n",
181 (unsigned long long)mask);
187 printf("BAR%d: Created Fixed mapping\n", bar->index);
188 printf("\t%d:%d:%d:0x%#llx-0x%#llx>\n", tgt, act, tok,
189 (unsigned long long)offset,
190 (unsigned long long)(offset + mask));
195 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
199 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE
200 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK);
202 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(tgt);
203 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(tok);
205 if ((offset & mask) != ((offset + size - 1) & mask)) {
206 printf("BAR%d: Won't use for bulk mapping\n",
208 printf("\t<%#llx,%#llx>\n", (unsigned long long)offset,
209 (unsigned long long)(offset + size));
210 printf("\ttarget=%d, token=%d\n", tgt, tok);
211 printf("\tBAR too small (%#llx) - (%#llx != %#llx).\n",
212 (unsigned long long)mask,
213 (unsigned long long)(offset & mask),
214 (unsigned long long)(offset + size - 1) & mask);
222 printf("BAR%d: Created bulk mapping %d:x:%d:%#llx-%#llx\n",
223 bar->index, tgt, tok, (unsigned long long)offset,
224 (unsigned long long)(offset + ~mask));
230 if (bar->bitsize < bitsize) {
231 printf("BAR%d: Too small for %d:%d:%d\n", bar->index, tgt, tok,
236 newcfg |= offset >> bitsize;
242 *bar_config = newcfg;
248 nfp_bar_write(struct nfp_pcie_user *nfp, struct nfp_bar *bar,
253 base = bar->index >> 3;
254 slot = bar->index & 7;
259 bar->csr = nfp->cfg +
260 NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(base, slot);
262 *(uint32_t *)(bar->csr) = newcfg;
264 bar->barcfg = newcfg;
266 printf("BAR%d: updated to 0x%08x\n", bar->index, newcfg);
273 nfp_reconfigure_bar(struct nfp_pcie_user *nfp, struct nfp_bar *bar, int tgt,
274 int act, int tok, uint64_t offset, size_t size, int width)
280 err = nfp_compute_bar(bar, &newcfg, &newbase, tgt, act, tok, offset,
287 return nfp_bar_write(nfp, bar, newcfg);
291 * Map all PCI bars. We assume that the BAR with the PCIe config block is
294 * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
297 nfp_enable_bars(struct nfp_pcie_user *nfp)
302 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
303 bar = &nfp->bar[x - 1];
307 bar->mask = (1 << (nfp->barsz - 3)) - 1;
308 bar->bitsize = nfp->barsz - 3;
312 bar->csr = nfp->cfg +
313 NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar->index >> 3,
316 bar->iomem = nfp->cfg + (bar->index << bar->bitsize);
321 static struct nfp_bar *
322 nfp_alloc_bar(struct nfp_pcie_user *nfp)
327 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
328 bar = &nfp->bar[x - 1];
338 nfp_disable_bars(struct nfp_pcie_user *nfp)
343 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
344 bar = &nfp->bar[x - 1];
353 * Generic CPP bus access interface.
356 struct nfp6000_area_priv {
374 nfp6000_area_init(struct nfp_cpp_area *area, uint32_t dest,
375 unsigned long long address, unsigned long size)
377 struct nfp_pcie_user *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
378 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
379 uint32_t target = NFP_CPP_ID_TARGET_of(dest);
380 uint32_t action = NFP_CPP_ID_ACTION_of(dest);
381 uint32_t token = NFP_CPP_ID_TOKEN_of(dest);
384 pp = nfp6000_target_pushpull(NFP_CPP_ID(target, action, token),
389 priv->width.read = PUSH_WIDTH(pp);
390 priv->width.write = PULL_WIDTH(pp);
392 if (priv->width.read > 0 &&
393 priv->width.write > 0 && priv->width.read != priv->width.write)
396 if (priv->width.read > 0)
397 priv->width.bar = priv->width.read;
399 priv->width.bar = priv->width.write;
401 priv->bar = nfp_alloc_bar(nfp);
402 if (priv->bar == NULL)
405 priv->target = target;
406 priv->action = action;
408 priv->offset = address;
411 ret = nfp_reconfigure_bar(nfp, priv->bar, priv->target, priv->action,
412 priv->token, priv->offset, priv->size,
419 nfp6000_area_acquire(struct nfp_cpp_area *area)
421 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
423 /* Calculate offset into BAR. */
424 if (nfp_bar_maptype(priv->bar) ==
425 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL) {
426 priv->bar_offset = priv->offset &
427 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
429 NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(priv->bar,
432 NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(priv->bar, priv->token);
434 priv->bar_offset = priv->offset & priv->bar->mask;
437 /* Must have been too big. Sub-allocate. */
438 if (!priv->bar->iomem)
441 priv->iomem = priv->bar->iomem + priv->bar_offset;
447 nfp6000_area_mapped(struct nfp_cpp_area *area)
449 struct nfp6000_area_priv *area_priv = nfp_cpp_area_priv(area);
451 if (!area_priv->iomem)
454 return area_priv->iomem;
458 nfp6000_area_release(struct nfp_cpp_area *area)
460 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
467 nfp6000_area_iomem(struct nfp_cpp_area *area)
469 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
474 nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
475 unsigned long offset, unsigned int length)
477 uint64_t *wrptr64 = kernel_vaddr;
478 const volatile uint64_t *rdptr64;
479 struct nfp6000_area_priv *priv;
480 uint32_t *wrptr32 = kernel_vaddr;
481 const volatile uint32_t *rdptr32;
486 priv = nfp_cpp_area_priv(area);
487 rdptr64 = (uint64_t *)(priv->iomem + offset);
488 rdptr32 = (uint32_t *)(priv->iomem + offset);
490 if (offset + length > priv->size)
493 width = priv->width.read;
498 /* Unaligned? Translate to an explicit access */
499 if ((priv->offset + offset) & (width - 1)) {
500 printf("aread_read unaligned!!!\n");
504 is_64 = width == TARGET_WIDTH_64;
506 /* MU reads via a PCIe2CPP BAR supports 32bit (and other) lengths */
507 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
508 priv->action == NFP_CPP_ACTION_RW) {
513 if (offset % sizeof(uint64_t) != 0 ||
514 length % sizeof(uint64_t) != 0)
517 if (offset % sizeof(uint32_t) != 0 ||
518 length % sizeof(uint32_t) != 0)
526 for (n = 0; n < length; n += sizeof(uint64_t)) {
532 for (n = 0; n < length; n += sizeof(uint32_t)) {
542 nfp6000_area_write(struct nfp_cpp_area *area, const void *kernel_vaddr,
543 unsigned long offset, unsigned int length)
545 const uint64_t *rdptr64 = kernel_vaddr;
547 const uint32_t *rdptr32 = kernel_vaddr;
548 struct nfp6000_area_priv *priv;
554 priv = nfp_cpp_area_priv(area);
555 wrptr64 = (uint64_t *)(priv->iomem + offset);
556 wrptr32 = (uint32_t *)(priv->iomem + offset);
558 if (offset + length > priv->size)
561 width = priv->width.write;
566 /* Unaligned? Translate to an explicit access */
567 if ((priv->offset + offset) & (width - 1))
570 is_64 = width == TARGET_WIDTH_64;
572 /* MU writes via a PCIe2CPP BAR supports 32bit (and other) lengths */
573 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
574 priv->action == NFP_CPP_ACTION_RW)
578 if (offset % sizeof(uint64_t) != 0 ||
579 length % sizeof(uint64_t) != 0)
582 if (offset % sizeof(uint32_t) != 0 ||
583 length % sizeof(uint32_t) != 0)
591 for (n = 0; n < length; n += sizeof(uint64_t)) {
597 for (n = 0; n < length; n += sizeof(uint32_t)) {
606 #define PCI_DEVICES "/sys/bus/pci/devices"
609 nfp_acquire_process_lock(struct nfp_pcie_user *desc)
615 memset(&lock, 0, sizeof(lock));
617 snprintf(lockname, sizeof(lockname), "/var/lock/nfp_%s", desc->busdev);
618 desc->lock = open(lockname, O_RDWR | O_CREAT, 0666);
622 lock.l_type = F_WRLCK;
623 lock.l_whence = SEEK_SET;
626 rc = fcntl(desc->lock, F_SETLKW, &lock);
628 if (errno != EAGAIN && errno != EACCES) {
639 nfp6000_set_model(struct rte_pci_device *dev, struct nfp_cpp *cpp)
643 if (rte_pci_read_config(dev, &model, 4, 0x2e) < 0) {
644 printf("nfp set model failed\n");
649 nfp_cpp_model_set(cpp, model);
655 nfp6000_set_interface(struct rte_pci_device *dev, struct nfp_cpp *cpp)
659 if (rte_pci_read_config(dev, &interface, 2, 0x154) < 0) {
660 printf("nfp set interface failed\n");
664 nfp_cpp_interface_set(cpp, interface);
669 #define PCI_CFG_SPACE_SIZE 256
670 #define PCI_CFG_SPACE_EXP_SIZE 4096
671 #define PCI_EXT_CAP_ID(header) (int)(header & 0x0000ffff)
672 #define PCI_EXT_CAP_NEXT(header) ((header >> 20) & 0xffc)
673 #define PCI_EXT_CAP_ID_DSN 0x03
675 nfp_pci_find_next_ext_capability(struct rte_pci_device *dev, int cap)
679 int pos = PCI_CFG_SPACE_SIZE;
681 /* minimum 8 bytes per capability */
682 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
684 if (rte_pci_read_config(dev, &header, 4, pos) < 0) {
685 printf("nfp error reading extended capabilities\n");
690 * If we have no capabilities, this is indicated by cap ID,
691 * cap version and next pointer all being 0.
697 if (PCI_EXT_CAP_ID(header) == cap)
700 pos = PCI_EXT_CAP_NEXT(header);
701 if (pos < PCI_CFG_SPACE_SIZE)
704 if (rte_pci_read_config(dev, &header, 4, pos) < 0) {
705 printf("nfp error reading extended capabilities\n");
714 nfp6000_set_serial(struct rte_pci_device *dev, struct nfp_cpp *cpp)
721 pos = nfp_pci_find_next_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
723 printf("PCI_EXT_CAP_ID_DSN not found. nfp set serial failed\n");
729 if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
730 printf("nfp set serial failed\n");
734 serial[4] = (uint8_t)((tmp >> 8) & 0xff);
735 serial[5] = (uint8_t)(tmp & 0xff);
738 if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
739 printf("nfp set serial failed\n");
743 serial[2] = (uint8_t)((tmp >> 8) & 0xff);
744 serial[3] = (uint8_t)(tmp & 0xff);
747 if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
748 printf("nfp set serial failed\n");
752 serial[0] = (uint8_t)((tmp >> 8) & 0xff);
753 serial[1] = (uint8_t)(tmp & 0xff);
755 nfp_cpp_serial_set(cpp, serial, serial_len);
761 nfp6000_set_barsz(struct rte_pci_device *dev, struct nfp_pcie_user *desc)
766 tmp = dev->mem_resource[0].len;
776 nfp6000_init(struct nfp_cpp *cpp, struct rte_pci_device *dev)
780 struct nfp_pcie_user *desc;
782 desc = malloc(sizeof(*desc));
787 memset(desc->busdev, 0, BUSDEV_SZ);
788 strlcpy(desc->busdev, dev->device.name, sizeof(desc->busdev));
790 if (cpp->driver_lock_needed) {
791 ret = nfp_acquire_process_lock(desc);
796 if (nfp6000_set_model(dev, cpp) < 0)
798 if (nfp6000_set_interface(dev, cpp) < 0)
800 if (nfp6000_set_serial(dev, cpp) < 0)
802 if (nfp6000_set_barsz(dev, desc) < 0)
805 desc->cfg = (char *)dev->mem_resource[0].addr;
807 nfp_enable_bars(desc);
809 nfp_cpp_priv_set(cpp, desc);
811 model = __nfp_cpp_model_autodetect(cpp);
812 nfp_cpp_model_set(cpp, model);
822 nfp6000_free(struct nfp_cpp *cpp)
824 struct nfp_pcie_user *desc = nfp_cpp_priv(cpp);
826 nfp_disable_bars(desc);
827 if (cpp->driver_lock_needed)
833 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
834 .init = nfp6000_init,
835 .free = nfp6000_free,
837 .area_priv_size = sizeof(struct nfp6000_area_priv),
838 .area_init = nfp6000_area_init,
839 .area_acquire = nfp6000_area_acquire,
840 .area_release = nfp6000_area_release,
841 .area_mapped = nfp6000_area_mapped,
842 .area_read = nfp6000_area_read,
843 .area_write = nfp6000_area_write,
844 .area_iomem = nfp6000_area_iomem,
848 nfp_cpp_operations *nfp_cpp_transport_operations(void)
850 return &nfp6000_pcie_ops;