X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fbuffer.c;h=ee63f76b0d4f57e6f952993891017bed042becb0;hb=671e60e65635b8d030bf303c88411192c747b59e;hp=6afd20e06fefce390b40414eed76066244923371;hpb=0e969ac8431c80ff4bca5f6985876b1c584eefcd;p=vpp.git diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 6afd20e06fe..ee63f76b0d4 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -44,8 +44,6 @@ */ #include -#include -#include #include @@ -64,7 +62,7 @@ #include #include #include -#include +#include #include #include #include @@ -76,8 +74,6 @@ #include #include -#include -#include #include #include #include @@ -144,7 +140,7 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) vec_free (f->name); vec_free (f->buffers); /* Poison it. */ - memset (f, 0xab, sizeof (f[0])); + clib_memset (f, 0xab, sizeof (f[0])); } /* Add buffer free list. */ @@ -213,7 +209,7 @@ CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm, if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0) return 0; - memset (&bt, 0, sizeof (vlib_buffer_t)); + clib_memset (&bt, 0, sizeof (vlib_buffer_t)); vlib_buffer_init_for_free_list (&bt, fl); bt.buffer_pool_index = privp->buffer_pool_index; @@ -280,30 +276,9 @@ static_always_inline void recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, vlib_buffer_t * b) { - vlib_buffer_free_list_t *fl; u32 thread_index = vlib_get_thread_index (); - vlib_buffer_free_list_index_t fi; - fl = vlib_buffer_get_buffer_free_list (vm, b, &fi); - /* The only current use of this callback: multicast recycle */ - if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) - { - int j; - - vlib_buffer_add_to_free_list (vm, fl, bi, - (b->flags & VLIB_BUFFER_RECYCLE) == 0); - - for (j = 0; j < vec_len (vm->buffer_announce_list); j++) - { - if (fl == vm->buffer_announce_list[j]) - goto already_announced; - } - vec_add1 (vm->buffer_announce_list, fl); - already_announced: - ; - } - else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) - dpdk_rte_pktmbuf_free (vm, thread_index, b, 1); + dpdk_rte_pktmbuf_free (vm, thread_index, b, 1); } static_always_inline void @@ -314,7 +289,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, vlib_buffer_t *bufp[n_buffers], **b = bufp; u32 thread_index = vlib_get_thread_index (); int i = 0; - u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE | + u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_NEXT_PRESENT); u32 n_left, *bi; u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, @@ -379,16 +354,6 @@ vlib_buffer_free_inline (vlib_main_t * vm, b += 1; n_left -= 1; } - if (vec_len (vm->buffer_announce_list)) - { - vlib_buffer_free_list_t *fl; - for (i = 0; i < vec_len (vm->buffer_announce_list); i++) - { - fl = vm->buffer_announce_list[i]; - fl->buffers_added_to_freelist_function (vm, fl); - } - _vec_len (vm->buffer_announce_list) = 0; - } } void @@ -408,79 +373,22 @@ CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers, } #ifndef CLIB_MARCH_VARIANT -static void -dpdk_packet_template_init (vlib_main_t * vm, - void *vt, - void *packet_data, - uword n_packet_data_bytes, - uword min_n_buffers_each_alloc, u8 * name) -{ - vlib_packet_template_t *t = (vlib_packet_template_t *) vt; - - vlib_worker_thread_barrier_sync (vm); - memset (t, 0, sizeof (t[0])); - - vec_add (t->packet_data, packet_data, n_packet_data_bytes); - - vlib_worker_thread_barrier_release (vm); -} - -static clib_error_t * -scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name) -{ - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - linux_vfio_main_t *lvm = &vfio_main; - const char fn[] = "/dev/vfio/vfio"; - char buff[sizeof (fn)] = { 0 }; - int fd; - u8 *path = format (0, "%v%c", path_name, 0); - - if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn)) - goto done; - - if (strncmp (fn, buff, sizeof (fn))) - goto done; - - fd = atoi ((char *) file_name); - if (fd != lvm->container_fd) - dbm->vfio_container_fd = fd; - -done: - vec_free (path); - return 0; -} - clib_error_t * dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, - struct rte_mempool ** _mp, - vlib_physmem_region_index_t * pri) + struct rte_mempool **_mp, u32 * map_index) { - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; struct rte_mempool *mp; - vlib_physmem_region_t *pr; + enum rte_iova_mode iova_mode; dpdk_mempool_private_t priv; + vlib_physmem_map_t *pm; clib_error_t *error = 0; - u32 size, obj_size; + size_t min_chunk_size, align; + int map_dma = 1; + u32 size; i32 ret; uword i; - obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0); - -#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0) - size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0); -#else - size = rte_mempool_calc_mem_size_helper (num_elts, obj_size, 21); -#endif - - error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa, - VLIB_PHYSMEM_F_HUGETLB | - VLIB_PHYSMEM_F_SHARED, pri); - if (error) - return error; - - pr = vlib_physmem_get_region (vm, pri[0]); - mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size, 512, pool_priv_size, numa, 0); if (!mp) @@ -488,53 +396,47 @@ dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); + size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21, + &min_chunk_size, &align); + + if ((error = vlib_physmem_shared_map_create (vm, (char *) pool_name, size, + 0, numa, map_index))) + { + rte_mempool_free (mp); + return error; + } + pm = vlib_physmem_get_map (vm, *map_index); + /* Call the mempool priv initializer */ priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE; priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; rte_pktmbuf_pool_init (mp, &priv); - for (i = 0; i < pr->n_pages; i++) + if (rte_eth_dev_count_avail () == 0) + map_dma = 0; + + iova_mode = rte_eal_iova_mode (); + for (i = 0; i < pm->n_pages; i++) { - size_t page_size = 1 << pr->log2_page_size; - ret = rte_mempool_populate_iova (mp, ((char *) pr->mem) + i * page_size, - pr->page_table[i], page_size, 0, 0); + size_t page_sz = 1ULL << pm->log2_page_size; + char *va = ((char *) pm->base) + i * page_sz; + uword pa = iova_mode == RTE_IOVA_VA ? + pointer_to_uword (va) : pm->page_table[i]; + ret = rte_mempool_populate_iova (mp, va, pa, page_sz, 0, 0); if (ret < 0) { rte_mempool_free (mp); return clib_error_return (0, "failed to populate %s", pool_name); } + /* -1 likely means there is no PCI devices assigned to vfio + container or noiommu mode is used so we stop trying */ + if (map_dma && rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz)) + map_dma = 0; } _mp[0] = mp; - /* DPDK currently doesn't provide API to map DMA memory for empty mempool - so we are using this hack, will be nice to have at least API to get - VFIO container FD */ - if (dbm->vfio_container_fd == -1) - foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0); - - if (dbm->vfio_container_fd != -1) - { - struct vfio_iommu_type1_dma_map dm = { 0 }; - int i, rv = 0; - dm.argsz = sizeof (struct vfio_iommu_type1_dma_map); - dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; - - /* *INDENT-OFF* */ - vec_foreach_index (i, pr->page_table) - { - dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size); - dm.size = 1 << pr->log2_page_size; - dm.iova = pr->page_table[i]; - if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm))) - break; - } - /* *INDENT-ON* */ - if (rv != 0 && errno != EINVAL) - clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name); - } - return 0; } @@ -544,10 +446,10 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, { dpdk_main_t *dm = &dpdk_main; struct rte_mempool *rmp; - vlib_physmem_region_index_t pri; clib_error_t *error = 0; u8 *pool_name; u32 elt_size, i; + u32 map_index; vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); @@ -561,10 +463,9 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, VLIB_BUFFER_HDR_SIZE /* priv size */ + VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE; /*data room size */ - error = - dpdk_pool_create (vm, pool_name, elt_size, num_mbufs, - sizeof (dpdk_mempool_private_t), 512, socket_id, - &rmp, &pri); + error = dpdk_pool_create (vm, pool_name, elt_size, num_mbufs, + sizeof (dpdk_mempool_private_t), 512, socket_id, + &rmp, &map_index); vec_free (pool_name); @@ -574,7 +475,8 @@ dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0); dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - privp->buffer_pool_index = vlib_buffer_pool_create (vm, pri, 0); + privp->buffer_pool_index = + vlib_buffer_register_physmem_map (vm, map_index); dm->pktmbuf_pools[socket_id] = rmp; @@ -611,8 +513,8 @@ buffer_state_validation_init (vlib_main_t * vm) { void *oldheap; - vlib_buffer_state_heap = mheap_alloc (0, 10 << 20); - + vlib_buffer_state_heap = + mheap_alloc_with_lock (0, 10 << 20, 0 /* locked */ ); oldheap = clib_mem_set_heap (vlib_buffer_state_heap); vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword)); @@ -707,7 +609,6 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = { .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list, .vlib_buffer_free_cb = &dpdk_buffer_free, .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next, - .vlib_packet_template_init_cb = &dpdk_packet_template_init, .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list, }; /* *INDENT-ON* */