drivers/mempool/dpaa2/dpaa2_hw_mempool.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   5  *   Copyright 2016 NXP.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Freescale Semiconductor, Inc nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <unistd.h>
  35 #include <stdio.h>
  36 #include <sys/types.h>
  37 #include <string.h>
  38 #include <stdlib.h>
  39 #include <fcntl.h>
  40 #include <errno.h>
  41
  42 #include <rte_mbuf.h>
  43 #include <rte_ethdev.h>
  44 #include <rte_malloc.h>
  45 #include <rte_memcpy.h>
  46 #include <rte_string_fns.h>
  47 #include <rte_cycles.h>
  48 #include <rte_kvargs.h>
  49 #include <rte_dev.h>
  50
  51 #include <fslmc_logs.h>
  52 #include <mc/fsl_dpbp.h>
  53 #include <portal/dpaa2_hw_pvt.h>
  54 #include <portal/dpaa2_hw_dpio.h>
  55 #include "dpaa2_hw_mempool.h"
  56
  57 struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID];
  58 static struct dpaa2_bp_list *h_bp_list;
  59
  60 static int
  61 rte_hw_mbuf_create_pool(struct rte_mempool *mp)
  62 {
  63         struct dpaa2_bp_list *bp_list;
  64         struct dpaa2_dpbp_dev *avail_dpbp;
  65         struct dpaa2_bp_info *bp_info;
  66         struct dpbp_attr dpbp_attr;
  67         uint32_t bpid;
  68         int ret;
  69
  70         avail_dpbp = dpaa2_alloc_dpbp_dev();
  71
  72         if (!avail_dpbp) {
  73                 PMD_DRV_LOG(ERR, "DPAA2 resources not available");
  74                 return -ENOENT;
  75         }
  76
  77         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
  78                 ret = dpaa2_affine_qbman_swp();
  79                 if (ret) {
  80                         RTE_LOG(ERR, PMD, "Failure in affining portal\n");
  81                         goto err1;
  82                 }
  83         }
  84
  85         ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
  86         if (ret != 0) {
  87                 PMD_INIT_LOG(ERR, "Resource enable failure with"
  88                         " err code: %d\n", ret);
  89                 goto err1;
  90         }
  91
  92         ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW,
  93                                   avail_dpbp->token, &dpbp_attr);
  94         if (ret != 0) {
  95                 PMD_INIT_LOG(ERR, "Resource read failure with"
  96                              " err code: %d\n", ret);
  97                 goto err2;
  98         }
  99
 100         bp_info = rte_malloc(NULL,
 101                              sizeof(struct dpaa2_bp_info),
 102                              RTE_CACHE_LINE_SIZE);
 103         if (!bp_info) {
 104                 PMD_INIT_LOG(ERR, "No heap memory available for bp_info");
 105                 ret = -ENOMEM;
 106                 goto err2;
 107         }
 108
 109         /* Allocate the bp_list which will be added into global_bp_list */
 110         bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list),
 111                              RTE_CACHE_LINE_SIZE);
 112         if (!bp_list) {
 113                 PMD_INIT_LOG(ERR, "No heap memory available");
 114                 ret = -ENOMEM;
 115                 goto err3;
 116         }
 117
 118         /* Set parameters of buffer pool list */
 119         bp_list->buf_pool.num_bufs = mp->size;
 120         bp_list->buf_pool.size = mp->elt_size
 121                         - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp);
 122         bp_list->buf_pool.bpid = dpbp_attr.bpid;
 123         bp_list->buf_pool.h_bpool_mem = NULL;
 124         bp_list->buf_pool.dpbp_node = avail_dpbp;
 125         /* Identification for our offloaded pool_data structure */
 126         bp_list->dpaa2_ops_index = mp->ops_index;
 127         bp_list->next = h_bp_list;
 128         bp_list->mp = mp;
 129
 130         bpid = dpbp_attr.bpid;
 131
 132         rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf)
 133                                 + rte_pktmbuf_priv_size(mp);
 134         rte_dpaa2_bpid_info[bpid].bp_list = bp_list;
 135         rte_dpaa2_bpid_info[bpid].bpid = bpid;
 136
 137         rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid],
 138                    sizeof(struct dpaa2_bp_info));
 139         mp->pool_data = (void *)bp_info;
 140
 141         PMD_INIT_LOG(DEBUG, "BP List created for bpid =%d", dpbp_attr.bpid);
 142
 143         h_bp_list = bp_list;
 144         return 0;
 145 err3:
 146         rte_free(bp_info);
 147 err2:
 148         dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
 149 err1:
 150         dpaa2_free_dpbp_dev(avail_dpbp);
 151
 152         return ret;
 153 }
 154
 155 static void
 156 rte_hw_mbuf_free_pool(struct rte_mempool *mp)
 157 {
 158         struct dpaa2_bp_info *bpinfo;
 159         struct dpaa2_bp_list *bp;
 160         struct dpaa2_dpbp_dev *dpbp_node;
 161
 162         if (!mp->pool_data) {
 163                 PMD_DRV_LOG(ERR, "Not a valid dpaa22 pool");
 164                 return;
 165         }
 166
 167         bpinfo = (struct dpaa2_bp_info *)mp->pool_data;
 168         bp = bpinfo->bp_list;
 169         dpbp_node = bp->buf_pool.dpbp_node;
 170
 171         dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token);
 172
 173         if (h_bp_list == bp) {
 174                 h_bp_list = h_bp_list->next;
 175         } else { /* if it is not the first node */
 176                 struct dpaa2_bp_list *prev = h_bp_list, *temp;
 177                 temp = h_bp_list->next;
 178                 while (temp) {
 179                         if (temp == bp) {
 180                                 prev->next = temp->next;
 181                                 rte_free(bp);
 182                                 break;
 183                         }
 184                         prev = temp;
 185                         temp = temp->next;
 186                 }
 187         }
 188
 189         rte_free(mp->pool_data);
 190         dpaa2_free_dpbp_dev(dpbp_node);
 191 }
 192
 193 static void
 194 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused,
 195                         void * const *obj_table,
 196                         uint32_t bpid,
 197                         uint32_t meta_data_size,
 198                         int count)
 199 {
 200         struct qbman_release_desc releasedesc;
 201         struct qbman_swp *swp;
 202         int ret;
 203         int i, n;
 204         uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
 205
 206         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 207                 ret = dpaa2_affine_qbman_swp();
 208                 if (ret != 0) {
 209                         RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n");
 210                         return;
 211                 }
 212         }
 213         swp = DPAA2_PER_LCORE_PORTAL;
 214
 215         /* Create a release descriptor required for releasing
 216          * buffers into QBMAN
 217          */
 218         qbman_release_desc_clear(&releasedesc);
 219         qbman_release_desc_set_bpid(&releasedesc, bpid);
 220
 221         n = count % DPAA2_MBUF_MAX_ACQ_REL;
 222         if (unlikely(!n))
 223                 goto aligned;
 224
 225         /* convert mbuf to buffers for the remainder */
 226         for (i = 0; i < n ; i++) {
 227 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 228                 bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i])
 229                                 + meta_data_size;
 230 #else
 231                 bufs[i] = (uint64_t)obj_table[i] + meta_data_size;
 232 #endif
 233         }
 234
 235         /* feed them to bman */
 236         do {
 237                 ret = qbman_swp_release(swp, &releasedesc, bufs, n);
 238         } while (ret == -EBUSY);
 239
 240 aligned:
 241         /* if there are more buffers to free */
 242         while (n < count) {
 243                 /* convert mbuf to buffers */
 244                 for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) {
 245 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 246                         bufs[i] = (uint64_t)
 247                                   rte_mempool_virt2iova(obj_table[n + i])
 248                                   + meta_data_size;
 249 #else
 250                         bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size;
 251 #endif
 252                 }
 253
 254                 do {
 255                         ret = qbman_swp_release(swp, &releasedesc, bufs,
 256                                                 DPAA2_MBUF_MAX_ACQ_REL);
 257                 } while (ret == -EBUSY);
 258                 n += DPAA2_MBUF_MAX_ACQ_REL;
 259         }
 260 }
 261
 262 int
 263 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool,
 264                           void **obj_table, unsigned int count)
 265 {
 266 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
 267         static int alloc;
 268 #endif
 269         struct qbman_swp *swp;
 270         uint16_t bpid;
 271         uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
 272         int i, ret;
 273         unsigned int n = 0;
 274         struct dpaa2_bp_info *bp_info;
 275
 276         bp_info = mempool_to_bpinfo(pool);
 277
 278         if (!(bp_info->bp_list)) {
 279                 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
 280                 return -ENOENT;
 281         }
 282
 283         bpid = bp_info->bpid;
 284
 285         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 286                 ret = dpaa2_affine_qbman_swp();
 287                 if (ret != 0) {
 288                         RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n");
 289                         return ret;
 290                 }
 291         }
 292         swp = DPAA2_PER_LCORE_PORTAL;
 293
 294         while (n < count) {
 295                 /* Acquire is all-or-nothing, so we drain in 7s,
 296                  * then the remainder.
 297                  */
 298                 if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) {
 299                         ret = qbman_swp_acquire(swp, bpid, bufs,
 300                                                 DPAA2_MBUF_MAX_ACQ_REL);
 301                 } else {
 302                         ret = qbman_swp_acquire(swp, bpid, bufs,
 303                                                 count - n);
 304                 }
 305                 /* In case of less than requested number of buffers available
 306                  * in pool, qbman_swp_acquire returns 0
 307                  */
 308                 if (ret <= 0) {
 309                         PMD_TX_LOG(ERR, "Buffer acquire failed with"
 310                                    " err code: %d", ret);
 311                         /* The API expect the exact number of requested bufs */
 312                         /* Releasing all buffers allocated */
 313                         rte_dpaa2_mbuf_release(pool, obj_table, bpid,
 314                                            bp_info->meta_data_size, n);
 315                         return -ENOBUFS;
 316                 }
 317                 /* assigning mbuf from the acquired objects */
 318                 for (i = 0; (i < ret) && bufs[i]; i++) {
 319                         DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], uint64_t);
 320                         obj_table[n] = (struct rte_mbuf *)
 321                                        (bufs[i] - bp_info->meta_data_size);
 322                         PMD_TX_LOG(DEBUG, "Acquired %p address %p from BMAN",
 323                                    (void *)bufs[i], (void *)obj_table[n]);
 324                         n++;
 325                 }
 326         }
 327
 328 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
 329         alloc += n;
 330         PMD_TX_LOG(DEBUG, "Total = %d , req = %d done = %d",
 331                    alloc, count, n);
 332 #endif
 333         return 0;
 334 }
 335
 336 static int
 337 rte_hw_mbuf_free_bulk(struct rte_mempool *pool,
 338                   void * const *obj_table, unsigned int n)
 339 {
 340         struct dpaa2_bp_info *bp_info;
 341
 342         bp_info = mempool_to_bpinfo(pool);
 343         if (!(bp_info->bp_list)) {
 344                 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
 345                 return -ENOENT;
 346         }
 347         rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid,
 348                            bp_info->meta_data_size, n);
 349
 350         return 0;
 351 }
 352
 353 static unsigned int
 354 rte_hw_mbuf_get_count(const struct rte_mempool *mp)
 355 {
 356         int ret;
 357         unsigned int num_of_bufs = 0;
 358         struct dpaa2_bp_info *bp_info;
 359         struct dpaa2_dpbp_dev *dpbp_node;
 360
 361         if (!mp || !mp->pool_data) {
 362                 RTE_LOG(ERR, PMD, "Invalid mempool provided\n");
 363                 return 0;
 364         }
 365
 366         bp_info = (struct dpaa2_bp_info *)mp->pool_data;
 367         dpbp_node = bp_info->bp_list->buf_pool.dpbp_node;
 368
 369         ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW,
 370                                      dpbp_node->token, &num_of_bufs);
 371         if (ret) {
 372                 RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)\n",
 373                         ret);
 374                 return 0;
 375         }
 376
 377         RTE_LOG(DEBUG, PMD, "Free bufs = %u\n", num_of_bufs);
 378
 379         return num_of_bufs;
 380 }
 381
 382 struct rte_mempool_ops dpaa2_mpool_ops = {
 383         .name = "dpaa2",
 384         .alloc = rte_hw_mbuf_create_pool,
 385         .free = rte_hw_mbuf_free_pool,
 386         .enqueue = rte_hw_mbuf_free_bulk,
 387         .dequeue = rte_dpaa2_mbuf_alloc_bulk,
 388         .get_count = rte_hw_mbuf_get_count,
 389 };
 390
 391 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops);