X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fbuffer.c;h=3324c04951144ec960fb17dea043b4347719cdbb;hb=refs%2Fchanges%2F73%2F12773%2F4;hp=a73d3ff32b1225663a47f326f8cf1683e2887c55;hpb=2d36ed2fac5c8eea5311e0eb8a3aaa7dc75f59ee;p=vpp.git diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index a73d3ff32b1..3324c049511 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -43,6 +43,10 @@ * Allocate/free network buffers. */ +#include +#include +#include + #include #include @@ -71,11 +75,13 @@ #include #include +#include +#include +#include #include #include #include - STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM, "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM"); @@ -91,6 +97,7 @@ typedef struct typedef struct { + int vfio_container_fd; dpdk_buffer_per_thread_data *ptd; } dpdk_buffer_main_t; @@ -139,7 +146,7 @@ next: } } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static void del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) { @@ -155,42 +162,32 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) vec_free (f->name); vec_free (f->buffers); + /* Poison it. */ + memset (f, 0xab, sizeof (f[0])); } /* Add buffer free list. */ static void -dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) +dpdk_buffer_delete_free_list (vlib_main_t * vm, + vlib_buffer_free_list_index_t free_list_index) { - vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; - u32 merge_index; int i; ASSERT (vlib_get_thread_index () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); - merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); - if (merge_index != ~0 && merge_index != free_list_index) - { - vlib_buffer_merge_free_lists (pool_elt_at_index - (bm->buffer_free_list_pool, merge_index), - f); - } - del_free_list (vm, f); - /* Poison it. */ - memset (f, 0xab, sizeof (f[0])); - - pool_put (bm->buffer_free_list_pool, f); + pool_put (vm->buffer_free_list_pool, f); for (i = 1; i < vec_len (vlib_mains); i++) { - bm = vlib_mains[i]->buffer_main; - f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);; - memset (f, 0xab, sizeof (f[0])); - pool_put (bm->buffer_free_list_pool, f); + vlib_main_t *wvm = vlib_mains[i]; + f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index); + del_free_list (wvm, f); + pool_put (wvm->buffer_free_list_pool, f); } } #endif @@ -224,7 +221,7 @@ CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm, n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32)); /* Always allocate new buffers in reasonably large sized chunks. */ - n = clib_max (n, fl->min_n_buffers_each_physmem_alloc); + n = clib_max (n, fl->min_n_buffers_each_alloc); vec_validate_aligned (vm->mbuf_alloc_list, n - 1, CLIB_CACHE_LINE_BYTES); @@ -321,7 +318,7 @@ recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, { vlib_buffer_free_list_t *fl; u32 thread_index = vlib_get_thread_index (); - u32 fi; + vlib_buffer_free_list_index_t fi; fl = vlib_buffer_get_buffer_free_list (vm, b, &fi); /* The only current use of this callback: multicast recycle */ @@ -332,12 +329,12 @@ recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, vlib_buffer_add_to_free_list (vm, fl, bi, (b->flags & VLIB_BUFFER_RECYCLE) == 0); - for (j = 0; j < vec_len (bm->announce_list); j++) + for (j = 0; j < vec_len (vm->buffer_announce_list); j++) { - if (fl == bm->announce_list[j]) + if (fl == vm->buffer_announce_list[j]) goto already_announced; } - vec_add1 (bm->announce_list, fl); + vec_add1 (vm->buffer_announce_list, fl); already_announced: ; } @@ -352,7 +349,7 @@ static_always_inline void vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next) { - vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_main_t *bm = &buffer_main; dpdk_buffer_main_t *dbm = &dpdk_buffer_main; vlib_buffer_t *b0, *b1, *b2, *b3; u32 thread_index = vlib_get_thread_index (); @@ -400,15 +397,15 @@ vlib_buffer_free_inline (vlib_main_t * vm, recycle_or_free (vm, bm, buffers[i], b0); i++; } - if (vec_len (bm->announce_list)) + if (vec_len (vm->buffer_announce_list)) { vlib_buffer_free_list_t *fl; - for (i = 0; i < vec_len (bm->announce_list); i++) + for (i = 0; i < vec_len (vm->buffer_announce_list); i++) { - fl = bm->announce_list[i]; + fl = vm->buffer_announce_list[i]; fl->buffers_added_to_freelist_function (vm, fl); } - _vec_len (bm->announce_list) = 0; + _vec_len (vm->buffer_announce_list) = 0; } vec_foreach_index (i, d->mbuf_pending_free_list) @@ -439,13 +436,13 @@ CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers, 0); } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static void dpdk_packet_template_init (vlib_main_t * vm, void *vt, void *packet_data, uword n_packet_data_bytes, - uword min_n_buffers_each_physmem_alloc, u8 * name) + uword min_n_buffers_each_alloc, u8 * name) { vlib_packet_template_t *t = (vlib_packet_template_t *) vt; @@ -457,26 +454,52 @@ dpdk_packet_template_init (vlib_main_t * vm, vlib_worker_thread_barrier_release (vm); } +static clib_error_t * +scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name) +{ + dpdk_buffer_main_t *dbm = &dpdk_buffer_main; + linux_vfio_main_t *lvm = &vfio_main; + const char fn[] = "/dev/vfio/vfio"; + char buff[sizeof (fn)] = { 0 }; + int fd; + u8 *path = format (0, "%v%c", path_name, 0); + + if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn)) + goto done; + + if (strncmp (fn, buff, sizeof (fn))) + goto done; + + fd = atoi ((char *) file_name); + if (fd != lvm->container_fd) + dbm->vfio_container_fd = fd; + +done: + vec_free (path); + return 0; +} + clib_error_t * dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, - struct rte_mempool **_mp, vlib_physmem_region_index_t * pri) + struct rte_mempool ** _mp, + vlib_physmem_region_index_t * pri) { + dpdk_buffer_main_t *dbm = &dpdk_buffer_main; struct rte_mempool *mp; vlib_physmem_region_t *pr; + dpdk_mempool_private_t priv; clib_error_t *error = 0; u32 size, obj_size; i32 ret; obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0); -#if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0) - size = rte_mempool_xmem_size (num_elts, obj_size, 21); -#else size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0); -#endif error = - vlib_physmem_region_alloc (vm, (i8 *) pool_name, size, numa, 0, pri); + vlib_physmem_region_alloc (vm, (i8 *) pool_name, size, numa, + VLIB_PHYSMEM_F_HUGETLB | VLIB_PHYSMEM_F_SHARED, + pri); if (error) return error; @@ -490,15 +513,15 @@ dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); -#if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0) - ret = - rte_mempool_populate_phys_tab (mp, pr->mem, pr->page_table, pr->n_pages, - pr->log2_page_size, NULL, NULL); -#else + /* Call the mempool priv initializer */ + priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + + VLIB_BUFFER_DATA_SIZE; + priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; + rte_pktmbuf_pool_init (mp, &priv); + ret = rte_mempool_populate_iova_tab (mp, pr->mem, pr->page_table, pr->n_pages, pr->log2_page_size, NULL, NULL); -#endif if (ret != (i32) mp->size) { rte_mempool_free (mp); @@ -507,6 +530,36 @@ dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, _mp[0] = mp; + /* DPDK currently doesn't provide API to map DMA memory for empty mempool + so we are using this hack, will be nice to have at least API to get + VFIO container FD */ + if (dbm->vfio_container_fd == -1) + foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0); + + if (dbm->vfio_container_fd != -1) + { + struct vfio_iommu_type1_dma_map dm = { 0 }; + int i, rv = 0; + dm.argsz = sizeof (struct vfio_iommu_type1_dma_map); + dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + /* *INDENT-OFF* */ + vec_foreach_index (i, pr->page_table) + { + dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size); + dm.size = 1 << pr->log2_page_size; + if (rte_eal_iova_mode() == RTE_IOVA_PA) + dm.iova = pr->page_table[i]; + else + dm.iova = dm.vaddr; + if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm))) + break; + } + /* *INDENT-ON* */ + if (rv != 0 && errno != EINVAL) + clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name); + } + return 0; } @@ -516,7 +569,6 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, { dpdk_main_t *dm = &dpdk_main; struct rte_mempool *rmp; - dpdk_mempool_private_t priv; vlib_physmem_region_index_t pri; clib_error_t *error = 0; u8 *pool_name; @@ -543,18 +595,11 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, if (!error) { - priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + - VLIB_BUFFER_DATA_SIZE; - priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; - - /* call the mempool priv initializer */ - rte_pktmbuf_pool_init (rmp, &priv); - /* call the object initializers */ rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0); dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - privp->buffer_pool_index = vlib_buffer_add_physmem_region (vm, pri); + privp->buffer_pool_index = vlib_buffer_pool_create (vm, pri, 0); dm->pktmbuf_pools[socket_id] = rmp; @@ -671,8 +716,12 @@ dpdk_buffer_init (vlib_main_t * vm) { dpdk_buffer_main_t *dbm = &dpdk_buffer_main; vlib_thread_main_t *tm = vlib_get_thread_main (); + vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); + + dbm->vfio_container_fd = -1; + return 0; }