drivers/bus/fslmc/fslmc_bus.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright 2016,2018 NXP
   4  *
   5  */
   6
   7 #include <string.h>
   8 #include <dirent.h>
   9 #include <stdbool.h>
  10
  11 #include <rte_log.h>
  12 #include <rte_bus.h>
  13 #include <rte_eal_memconfig.h>
  14 #include <rte_malloc.h>
  15 #include <rte_devargs.h>
  16 #include <rte_memcpy.h>
  17 #include <rte_ethdev_driver.h>
  18
  19 #include <rte_fslmc.h>
  20 #include <fslmc_vfio.h>
  21 #include "fslmc_logs.h"
  22
  23 #include <dpaax_iova_table.h>
  24
  25 int dpaa2_logtype_bus;
  26
  27 #define VFIO_IOMMU_GROUP_PATH "/sys/kernel/iommu_groups"
  28 #define FSLMC_BUS_NAME  fslmc
  29
  30 struct rte_fslmc_bus rte_fslmc_bus;
  31 uint8_t dpaa2_virt_mode;
  32
  33 uint32_t
  34 rte_fslmc_get_device_count(enum rte_dpaa2_dev_type device_type)
  35 {
  36         if (device_type > DPAA2_DEVTYPE_MAX)
  37                 return 0;
  38         return rte_fslmc_bus.device_count[device_type];
  39 }
  40
  41 RTE_DEFINE_PER_LCORE(struct dpaa2_portal_dqrr, dpaa2_held_bufs);
  42
  43 static void
  44 cleanup_fslmc_device_list(void)
  45 {
  46         struct rte_dpaa2_device *dev;
  47         struct rte_dpaa2_device *t_dev;
  48
  49         TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, t_dev) {
  50                 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
  51                 free(dev);
  52                 dev = NULL;
  53         }
  54 }
  55
  56 static int
  57 compare_dpaa2_devname(struct rte_dpaa2_device *dev1,
  58                       struct rte_dpaa2_device *dev2)
  59 {
  60         int comp;
  61
  62         if (dev1->dev_type > dev2->dev_type) {
  63                 comp = 1;
  64         } else if (dev1->dev_type < dev2->dev_type) {
  65                 comp = -1;
  66         } else {
  67                 /* Check the ID as types match */
  68                 if (dev1->object_id > dev2->object_id)
  69                         comp = 1;
  70                 else if (dev1->object_id < dev2->object_id)
  71                         comp = -1;
  72                 else
  73                         comp = 0; /* Duplicate device name */
  74         }
  75
  76         return comp;
  77 }
  78
  79 static void
  80 insert_in_device_list(struct rte_dpaa2_device *newdev)
  81 {
  82         int comp, inserted = 0;
  83         struct rte_dpaa2_device *dev = NULL;
  84         struct rte_dpaa2_device *tdev = NULL;
  85
  86         TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, tdev) {
  87                 comp = compare_dpaa2_devname(newdev, dev);
  88                 if (comp < 0) {
  89                         TAILQ_INSERT_BEFORE(dev, newdev, next);
  90                         inserted = 1;
  91                         break;
  92                 }
  93         }
  94
  95         if (!inserted)
  96                 TAILQ_INSERT_TAIL(&rte_fslmc_bus.device_list, newdev, next);
  97 }
  98
  99 static struct rte_devargs *
 100 fslmc_devargs_lookup(struct rte_dpaa2_device *dev)
 101 {
 102         struct rte_devargs *devargs;
 103         char dev_name[32];
 104
 105         RTE_EAL_DEVARGS_FOREACH("fslmc", devargs) {
 106                 devargs->bus->parse(devargs->name, &dev_name);
 107                 if (strcmp(dev_name, dev->device.name) == 0) {
 108                         DPAA2_BUS_INFO("**Devargs matched %s", dev_name);
 109                         return devargs;
 110                 }
 111         }
 112         return NULL;
 113 }
 114
 115 static void
 116 dump_device_list(void)
 117 {
 118         struct rte_dpaa2_device *dev;
 119         uint32_t global_log_level;
 120         int local_log_level;
 121
 122         /* Only if the log level has been set to Debugging, print list */
 123         global_log_level = rte_log_get_global_level();
 124         local_log_level = rte_log_get_level(dpaa2_logtype_bus);
 125         if (global_log_level == RTE_LOG_DEBUG ||
 126             local_log_level == RTE_LOG_DEBUG) {
 127                 DPAA2_BUS_LOG(DEBUG, "List of devices scanned on bus:");
 128                 TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
 129                         DPAA2_BUS_LOG(DEBUG, "\t\t%s", dev->device.name);
 130                 }
 131         }
 132 }
 133
 134 static int
 135 scan_one_fslmc_device(char *dev_name)
 136 {
 137         char *dup_dev_name, *t_ptr;
 138         struct rte_dpaa2_device *dev;
 139
 140         if (!dev_name)
 141                 return -1;
 142
 143         /* Ignore the Container name itself */
 144         if (!strncmp("dprc", dev_name, 4))
 145                 return 0;
 146
 147         /* Creating a temporary copy to perform cut-parse over string */
 148         dup_dev_name = strdup(dev_name);
 149         if (!dup_dev_name) {
 150                 DPAA2_BUS_ERR("Unable to allocate device name memory");
 151                 return -ENOMEM;
 152         }
 153
 154         /* For all other devices, we allocate rte_dpaa2_device.
 155          * For those devices where there is no driver, probe would release
 156          * the memory associated with the rte_dpaa2_device after necessary
 157          * initialization.
 158          */
 159         dev = calloc(1, sizeof(struct rte_dpaa2_device));
 160         if (!dev) {
 161                 DPAA2_BUS_ERR("Unable to allocate device object");
 162                 free(dup_dev_name);
 163                 return -ENOMEM;
 164         }
 165
 166         dev->device.bus = &rte_fslmc_bus.bus;
 167
 168         /* Parse the device name and ID */
 169         t_ptr = strtok(dup_dev_name, ".");
 170         if (!t_ptr) {
 171                 DPAA2_BUS_ERR("Incorrect device name observed");
 172                 goto cleanup;
 173         }
 174         if (!strncmp("dpni", t_ptr, 4))
 175                 dev->dev_type = DPAA2_ETH;
 176         else if (!strncmp("dpseci", t_ptr, 6))
 177                 dev->dev_type = DPAA2_CRYPTO;
 178         else if (!strncmp("dpcon", t_ptr, 5))
 179                 dev->dev_type = DPAA2_CON;
 180         else if (!strncmp("dpbp", t_ptr, 4))
 181                 dev->dev_type = DPAA2_BPOOL;
 182         else if (!strncmp("dpio", t_ptr, 4))
 183                 dev->dev_type = DPAA2_IO;
 184         else if (!strncmp("dpci", t_ptr, 4))
 185                 dev->dev_type = DPAA2_CI;
 186         else if (!strncmp("dpmcp", t_ptr, 5))
 187                 dev->dev_type = DPAA2_MPORTAL;
 188         else if (!strncmp("dpdmai", t_ptr, 6))
 189                 dev->dev_type = DPAA2_QDMA;
 190         else
 191                 dev->dev_type = DPAA2_UNKNOWN;
 192
 193         /* Update the device found into the device_count table */
 194         rte_fslmc_bus.device_count[dev->dev_type]++;
 195
 196         t_ptr = strtok(NULL, ".");
 197         if (!t_ptr) {
 198                 DPAA2_BUS_ERR("Incorrect device string observed (null)");
 199                 goto cleanup;
 200         }
 201
 202         sscanf(t_ptr, "%hu", &dev->object_id);
 203         dev->device.name = strdup(dev_name);
 204         if (!dev->device.name) {
 205                 DPAA2_BUS_ERR("Unable to clone device name. Out of memory");
 206                 goto cleanup;
 207         }
 208         dev->device.devargs = fslmc_devargs_lookup(dev);
 209
 210         /* Add device in the fslmc device list */
 211         insert_in_device_list(dev);
 212
 213         /* Don't need the duplicated device filesystem entry anymore */
 214         if (dup_dev_name)
 215                 free(dup_dev_name);
 216
 217         return 0;
 218 cleanup:
 219         if (dup_dev_name)
 220                 free(dup_dev_name);
 221         if (dev)
 222                 free(dev);
 223         return -1;
 224 }
 225
 226 static int
 227 rte_fslmc_parse(const char *name, void *addr)
 228 {
 229         uint16_t dev_id;
 230         char *t_ptr = NULL, *dname = NULL;
 231
 232         /* 'name' is expected to contain name of device, for example, dpio.1,
 233          * dpni.2, etc.
 234          */
 235
 236         dname = strdup(name);
 237         if (!dname)
 238                 return -EINVAL;
 239         t_ptr = dname;
 240
 241         if (strncmp("dpni", t_ptr, 4) &&
 242             strncmp("dpseci", t_ptr, 6) &&
 243             strncmp("dpcon", t_ptr, 5) &&
 244             strncmp("dpbp", t_ptr, 4) &&
 245             strncmp("dpio", t_ptr, 4) &&
 246             strncmp("dpci", t_ptr, 4) &&
 247             strncmp("dpmcp", t_ptr, 5) &&
 248             strncmp("dpdmai", t_ptr, 6)) {
 249                 DPAA2_BUS_DEBUG("Unknown or unsupported device (%s)", name);
 250                 goto err_out;
 251         }
 252
 253         t_ptr = strchr(name, '.');
 254         if (!t_ptr) {
 255                 DPAA2_BUS_ERR("Incorrect device string observed (null)");
 256                 goto err_out;
 257         }
 258
 259         t_ptr = (char *)(t_ptr + 1);
 260         if (sscanf(t_ptr, "%hu", &dev_id) <= 0) {
 261                 DPAA2_BUS_ERR("Incorrect device string observed (%s)", t_ptr);
 262                 goto err_out;
 263         }
 264         free(dname);
 265
 266         if (addr)
 267                 strcpy(addr, name);
 268
 269         return 0;
 270 err_out:
 271         free(dname);
 272         return -EINVAL;
 273 }
 274
 275 static int
 276 rte_fslmc_scan(void)
 277 {
 278         int ret;
 279         int device_count = 0;
 280         char fslmc_dirpath[PATH_MAX];
 281         DIR *dir;
 282         struct dirent *entry;
 283         static int process_once;
 284         int groupid;
 285
 286         if (process_once) {
 287                 DPAA2_BUS_DEBUG("Fslmc bus already scanned. Not rescanning");
 288                 return 0;
 289         }
 290         process_once = 1;
 291
 292         ret = fslmc_get_container_group(&groupid);
 293         if (ret != 0)
 294                 goto scan_fail;
 295
 296         /* Scan devices on the group */
 297         snprintf(fslmc_dirpath, sizeof(fslmc_dirpath), "%s/%d/devices",
 298                         VFIO_IOMMU_GROUP_PATH, groupid);
 299         dir = opendir(fslmc_dirpath);
 300         if (!dir) {
 301                 DPAA2_BUS_ERR("Unable to open VFIO group directory");
 302                 goto scan_fail;
 303         }
 304
 305         while ((entry = readdir(dir)) != NULL) {
 306                 if (entry->d_name[0] == '.' || entry->d_type != DT_LNK)
 307                         continue;
 308
 309                 ret = scan_one_fslmc_device(entry->d_name);
 310                 if (ret != 0) {
 311                         /* Error in parsing directory - exit gracefully */
 312                         goto scan_fail_cleanup;
 313                 }
 314                 device_count += 1;
 315         }
 316
 317         closedir(dir);
 318
 319         DPAA2_BUS_INFO("FSLMC Bus scan completed");
 320         /* If debugging is enabled, device list is dumped to log output */
 321         dump_device_list();
 322
 323         return 0;
 324
 325 scan_fail_cleanup:
 326         closedir(dir);
 327
 328         /* Remove all devices in the list */
 329         cleanup_fslmc_device_list();
 330 scan_fail:
 331         DPAA2_BUS_DEBUG("FSLMC Bus Not Available. Skipping");
 332         /* Irrespective of failure, scan only return success */
 333         return 0;
 334 }
 335
 336 static int
 337 rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
 338                 struct rte_dpaa2_device *dpaa2_dev)
 339 {
 340         if (dpaa2_drv->drv_type == dpaa2_dev->dev_type)
 341                 return 0;
 342
 343         return 1;
 344 }
 345
 346 static int
 347 rte_fslmc_probe(void)
 348 {
 349         int ret = 0;
 350         int probe_all;
 351
 352         struct rte_dpaa2_device *dev;
 353         struct rte_dpaa2_driver *drv;
 354
 355         if (TAILQ_EMPTY(&rte_fslmc_bus.device_list))
 356                 return 0;
 357
 358         ret = fslmc_vfio_setup_group();
 359         if (ret) {
 360                 DPAA2_BUS_ERR("Unable to setup VFIO %d", ret);
 361                 return 0;
 362         }
 363
 364         /* Map existing segments as well as, in case of hotpluggable memory,
 365          * install callback handler.
 366          */
 367         ret = rte_fslmc_vfio_dmamap();
 368         if (ret) {
 369                 DPAA2_BUS_ERR("Unable to DMA map existing VAs: (%d)", ret);
 370                 /* Not continuing ahead */
 371                 DPAA2_BUS_ERR("FSLMC VFIO Mapping failed");
 372                 return 0;
 373         }
 374
 375         ret = fslmc_vfio_process_group();
 376         if (ret) {
 377                 DPAA2_BUS_ERR("Unable to setup devices %d", ret);
 378                 return 0;
 379         }
 380
 381         probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST;
 382
 383         /* In case of PA, the FD addresses returned by qbman APIs are physical
 384          * addresses, which need conversion into equivalent VA address for
 385          * rte_mbuf. For that, a table (a serial array, in memory) is used to
 386          * increase translation efficiency.
 387          * This has to be done before probe as some device initialization
 388          * (during) probe allocate memory (dpaa2_sec) which needs to be pinned
 389          * to this table.
 390          *
 391          * Error is ignored as relevant logs are handled within dpaax and
 392          * handling for unavailable dpaax table too is transparent to caller.
 393          */
 394         dpaax_iova_table_populate();
 395
 396         TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
 397                 TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
 398                         ret = rte_fslmc_match(drv, dev);
 399                         if (ret)
 400                                 continue;
 401
 402                         if (!drv->probe)
 403                                 continue;
 404
 405                         if (rte_dev_is_probed(&dev->device))
 406                                 continue;
 407
 408                         if (dev->device.devargs &&
 409                           dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
 410                                 DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
 411                                               dev->device.name);
 412                                 continue;
 413                         }
 414
 415                         if (probe_all ||
 416                            (dev->device.devargs &&
 417                            dev->device.devargs->policy ==
 418                            RTE_DEV_WHITELISTED)) {
 419                                 ret = drv->probe(drv, dev);
 420                                 if (ret) {
 421                                         DPAA2_BUS_ERR("Unable to probe");
 422                                 } else {
 423                                         dev->driver = drv;
 424                                         dev->device.driver = &drv->driver;
 425                                 }
 426                         }
 427                         break;
 428                 }
 429         }
 430
 431         if (rte_eal_iova_mode() == RTE_IOVA_VA)
 432                 dpaa2_virt_mode = 1;
 433
 434         return 0;
 435 }
 436
 437 static struct rte_device *
 438 rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 439                       const void *data)
 440 {
 441         const struct rte_dpaa2_device *dstart;
 442         struct rte_dpaa2_device *dev;
 443
 444         if (start != NULL) {
 445                 dstart = RTE_DEV_TO_FSLMC_CONST(start);
 446                 dev = TAILQ_NEXT(dstart, next);
 447         } else {
 448                 dev = TAILQ_FIRST(&rte_fslmc_bus.device_list);
 449         }
 450         while (dev != NULL) {
 451                 if (cmp(&dev->device, data) == 0)
 452                         return &dev->device;
 453                 dev = TAILQ_NEXT(dev, next);
 454         }
 455
 456         return NULL;
 457 }
 458
 459 /*register a fslmc bus based dpaa2 driver */
 460 void
 461 rte_fslmc_driver_register(struct rte_dpaa2_driver *driver)
 462 {
 463         RTE_VERIFY(driver);
 464
 465         TAILQ_INSERT_TAIL(&rte_fslmc_bus.driver_list, driver, next);
 466         /* Update Bus references */
 467         driver->fslmc_bus = &rte_fslmc_bus;
 468 }
 469
 470 /*un-register a fslmc bus based dpaa2 driver */
 471 void
 472 rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver)
 473 {
 474         struct rte_fslmc_bus *fslmc_bus;
 475
 476         fslmc_bus = driver->fslmc_bus;
 477
 478         /* Cleanup the PA->VA Translation table; From whereever this function
 479          * is called from.
 480          */
 481         dpaax_iova_table_depopulate();
 482
 483         TAILQ_REMOVE(&fslmc_bus->driver_list, driver, next);
 484         /* Update Bus references */
 485         driver->fslmc_bus = NULL;
 486 }
 487
 488 /*
 489  * All device has iova as va
 490  */
 491 static inline int
 492 fslmc_all_device_support_iova(void)
 493 {
 494         int ret = 0;
 495         struct rte_dpaa2_device *dev;
 496         struct rte_dpaa2_driver *drv;
 497
 498         TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
 499                 TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
 500                         ret = rte_fslmc_match(drv, dev);
 501                         if (ret)
 502                                 continue;
 503                         /* if the driver is not supporting IOVA */
 504                         if (!(drv->drv_flags & RTE_DPAA2_DRV_IOVA_AS_VA))
 505                                 return 0;
 506                 }
 507         }
 508         return 1;
 509 }
 510
 511 /*
 512  * Get iommu class of DPAA2 devices on the bus.
 513  */
 514 static enum rte_iova_mode
 515 rte_dpaa2_get_iommu_class(void)
 516 {
 517         bool is_vfio_noiommu_enabled = 1;
 518         bool has_iova_va;
 519
 520         if (TAILQ_EMPTY(&rte_fslmc_bus.device_list))
 521                 return RTE_IOVA_DC;
 522
 523 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 524         return RTE_IOVA_PA;
 525 #endif
 526
 527         /* check if all devices on the bus support Virtual addressing or not */
 528         has_iova_va = fslmc_all_device_support_iova();
 529
 530 #ifdef VFIO_PRESENT
 531         is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ?
 532                                                 true : false;
 533 #endif
 534
 535         if (has_iova_va && !is_vfio_noiommu_enabled)
 536                 return RTE_IOVA_VA;
 537
 538         return RTE_IOVA_PA;
 539 }
 540
 541 struct rte_fslmc_bus rte_fslmc_bus = {
 542         .bus = {
 543                 .scan = rte_fslmc_scan,
 544                 .probe = rte_fslmc_probe,
 545                 .parse = rte_fslmc_parse,
 546                 .find_device = rte_fslmc_find_device,
 547                 .get_iommu_class = rte_dpaa2_get_iommu_class,
 548         },
 549         .device_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.device_list),
 550         .driver_list = TAILQ_HEAD_INITIALIZER(rte_fslmc_bus.driver_list),
 551         .device_count = {0},
 552 };
 553
 554 RTE_REGISTER_BUS(FSLMC_BUS_NAME, rte_fslmc_bus.bus);
 555
 556 RTE_INIT(fslmc_init_log)
 557 {
 558         /* Bus level logs */
 559         dpaa2_logtype_bus = rte_log_register("bus.fslmc");
 560         if (dpaa2_logtype_bus >= 0)
 561                 rte_log_set_level(dpaa2_logtype_bus, RTE_LOG_NOTICE);
 562 }