X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fbuffer.c;h=cc7eed059f0f29ed30d4c8c7a97efccc61c5bfac;hb=b9ffffd61967ba9bc9453c93348be1ea5412c638;hp=3b3aaf2379f6434bc6e125ea143c21250ac4ced6;hpb=ffe9d21b07ce4b607fd9a774df04f9f1f38eb197;p=vpp.git diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 3b3aaf2379f..cc7eed059f0 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -44,8 +44,6 @@ */ #include -#include -#include #include @@ -64,7 +62,7 @@ #include #include #include -#include +#include #include #include #include @@ -76,8 +74,6 @@ #include #include -#include -#include #include #include #include @@ -88,11 +84,7 @@ STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM, typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - struct rte_mbuf ***mbuf_pending_free_list; - - /* cached last pool */ - struct rte_mempool *last_pool; - u8 last_buffer_pool_index; + struct rte_mbuf **mbuf_alloc_list; } dpdk_buffer_per_thread_data; typedef struct @@ -104,14 +96,11 @@ typedef struct dpdk_buffer_main_t dpdk_buffer_main; static_always_inline void -dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b) +dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b, + int maybe_next) { - vlib_buffer_t *hb = b; - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index); struct rte_mbuf *mb; u32 next, flags; - mb = rte_mbuf_from_vlib_buffer (hb); next: flags = b->flags; @@ -124,22 +113,10 @@ next: b->n_add_refs = 0; } - mb = rte_pktmbuf_prefree_seg (mb); - if (mb) - { - if (mb->pool != d->last_pool) - { - d->last_pool = mb->pool; - dpdk_mempool_private_t *privp = rte_mempool_get_priv (d->last_pool); - d->last_buffer_pool_index = privp->buffer_pool_index; - vec_validate_aligned (d->mbuf_pending_free_list, - d->last_buffer_pool_index, - CLIB_CACHE_LINE_BYTES); - } - vec_add1 (d->mbuf_pending_free_list[d->last_buffer_pool_index], mb); - } + if ((mb = rte_pktmbuf_prefree_seg (mb))) + rte_mempool_put (mb->pool, mb); - if (flags & VLIB_BUFFER_NEXT_PRESENT) + if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT)) { b = vlib_get_buffer (vm, next); goto next; @@ -157,7 +134,7 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) for (i = 0; i < vec_len (f->buffers); i++) { b = vlib_get_buffer (vm, f->buffers[i]); - dpdk_rte_pktmbuf_free (vm, thread_index, b); + dpdk_rte_pktmbuf_free (vm, thread_index, b, 1); } vec_free (f->name); @@ -199,116 +176,99 @@ CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm, uword min_free_buffers) { dpdk_main_t *dm = &dpdk_main; - vlib_buffer_t *b0, *b1, *b2, *b3; - int n, i; - u32 bi0, bi1, bi2, bi3; + dpdk_buffer_main_t *dbm = &dpdk_buffer_main; + struct rte_mbuf **mb; + uword n_left, first; + word n_alloc; unsigned socket_id = rte_socket_id (); + u32 thread_index = vlib_get_thread_index (); + dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index); struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id]; dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - struct rte_mbuf *mb0, *mb1, *mb2, *mb3; vlib_buffer_t bt; + u32 *bi; /* Too early? */ if (PREDICT_FALSE (rmp == 0)) return 0; /* Already have enough free buffers on free list? */ - n = min_free_buffers - vec_len (fl->buffers); - if (n <= 0) + n_alloc = min_free_buffers - vec_len (fl->buffers); + if (n_alloc <= 0) return min_free_buffers; /* Always allocate round number of buffers. */ - n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32)); + n_alloc = round_pow2 (n_alloc, CLIB_CACHE_LINE_BYTES / sizeof (u32)); /* Always allocate new buffers in reasonably large sized chunks. */ - n = clib_max (n, fl->min_n_buffers_each_alloc); + n_alloc = clib_max (n_alloc, fl->min_n_buffers_each_alloc); - vec_validate_aligned (vm->mbuf_alloc_list, n - 1, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (d->mbuf_alloc_list, n_alloc - 1, + CLIB_CACHE_LINE_BYTES); - if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0) + if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0) return 0; memset (&bt, 0, sizeof (vlib_buffer_t)); vlib_buffer_init_for_free_list (&bt, fl); bt.buffer_pool_index = privp->buffer_pool_index; - _vec_len (vm->mbuf_alloc_list) = n; + _vec_len (d->mbuf_alloc_list) = n_alloc; - i = 0; - int f = vec_len (fl->buffers); - vec_resize_aligned (fl->buffers, n, CLIB_CACHE_LINE_BYTES); + first = vec_len (fl->buffers); + vec_resize_aligned (fl->buffers, n_alloc, CLIB_CACHE_LINE_BYTES); - while (i < (n - 7)) - { - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf - (vm->mbuf_alloc_list[i + 4]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf - (vm->mbuf_alloc_list[i + 5]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf - (vm->mbuf_alloc_list[i + 6]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf - (vm->mbuf_alloc_list[i + 7]), STORE); - - mb0 = vm->mbuf_alloc_list[i]; - mb1 = vm->mbuf_alloc_list[i + 1]; - mb2 = vm->mbuf_alloc_list[i + 2]; - mb3 = vm->mbuf_alloc_list[i + 3]; - - b0 = vlib_buffer_from_rte_mbuf (mb0); - b1 = vlib_buffer_from_rte_mbuf (mb1); - b2 = vlib_buffer_from_rte_mbuf (mb2); - b3 = vlib_buffer_from_rte_mbuf (mb3); - - bi0 = vlib_get_buffer_index (vm, b0); - bi1 = vlib_get_buffer_index (vm, b1); - bi2 = vlib_get_buffer_index (vm, b2); - bi3 = vlib_get_buffer_index (vm, b3); - - fl->buffers[f++] = bi0; - fl->buffers[f++] = bi1; - fl->buffers[f++] = bi2; - fl->buffers[f++] = bi3; - - clib_memcpy64_x4 (b0, b1, b2, b3, &bt); - - if (fl->buffer_init_function) - { - fl->buffer_init_function (vm, fl, &bi0, 1); - fl->buffer_init_function (vm, fl, &bi1, 1); - fl->buffer_init_function (vm, fl, &bi2, 1); - fl->buffer_init_function (vm, fl, &bi3, 1); - } - i += 4; - } - - while (i < n) - { - mb0 = vm->mbuf_alloc_list[i]; - - b0 = vlib_buffer_from_rte_mbuf (mb0); - bi0 = vlib_get_buffer_index (vm, b0); + n_left = n_alloc; + mb = d->mbuf_alloc_list; + bi = fl->buffers + first; - fl->buffers[f++] = bi0; - clib_memcpy (b0, &bt, sizeof (vlib_buffer_t)); + ASSERT (n_left % 8 == 0); - if (fl->buffer_init_function) - fl->buffer_init_function (vm, fl, &bi0, 1); - i++; + while (n_left >= 8) + { + if (PREDICT_FALSE (n_left < 24)) + goto no_prefetch; + + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[16]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[17]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[18]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[19]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[20]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[21]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[22]), STORE); + vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[23]), STORE); + + no_prefetch: + vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8, + sizeof (struct rte_mbuf)); + clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[0]), + vlib_buffer_from_rte_mbuf (mb[1]), + vlib_buffer_from_rte_mbuf (mb[2]), + vlib_buffer_from_rte_mbuf (mb[3]), &bt); + clib_memcpy64_x4 (vlib_buffer_from_rte_mbuf (mb[4]), + vlib_buffer_from_rte_mbuf (mb[5]), + vlib_buffer_from_rte_mbuf (mb[6]), + vlib_buffer_from_rte_mbuf (mb[7]), &bt); + + n_left -= 8; + mb += 8; + bi += 8; } - fl->n_alloc += n; + if (fl->buffer_init_function) + fl->buffer_init_function (vm, fl, fl->buffers + first, n_alloc); - return n; + fl->n_alloc += n_alloc; + + return n_alloc; } static_always_inline void -dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi) +dpdk_prefetch_buffer (vlib_buffer_t * b) { - vlib_buffer_t *b; struct rte_mbuf *mb; - b = vlib_get_buffer (vm, bi); mb = rte_mbuf_from_vlib_buffer (b); - CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); } @@ -316,33 +276,9 @@ static_always_inline void recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, vlib_buffer_t * b) { - vlib_buffer_free_list_t *fl; u32 thread_index = vlib_get_thread_index (); - vlib_buffer_free_list_index_t fi; - fl = vlib_buffer_get_buffer_free_list (vm, b, &fi); - /* The only current use of this callback: multicast recycle */ - if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) - { - int j; - - vlib_buffer_add_to_free_list (vm, fl, bi, - (b->flags & VLIB_BUFFER_RECYCLE) == 0); - - for (j = 0; j < vec_len (vm->buffer_announce_list); j++) - { - if (fl == vm->buffer_announce_list[j]) - goto already_announced; - } - vec_add1 (vm->buffer_announce_list, fl); - already_announced: - ; - } - else - { - if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) - dpdk_rte_pktmbuf_free (vm, thread_index, b); - } + dpdk_rte_pktmbuf_free (vm, thread_index, b, 1); } static_always_inline void @@ -350,11 +286,12 @@ vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next) { vlib_buffer_main_t *bm = &buffer_main; - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - vlib_buffer_t *b0, *b1, *b2, *b3; + vlib_buffer_t *bufp[n_buffers], **b = bufp; u32 thread_index = vlib_get_thread_index (); - dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index); int i = 0; + u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | + VLIB_BUFFER_NEXT_PRESENT); + u32 n_left, *bi; u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next); @@ -366,58 +303,57 @@ vlib_buffer_free_inline (vlib_main_t * vm, if (!n_buffers) return; - while (i + 7 < n_buffers) - { - dpdk_prefetch_buffer_by_index (vm, buffers[i + 4]); - dpdk_prefetch_buffer_by_index (vm, buffers[i + 5]); - dpdk_prefetch_buffer_by_index (vm, buffers[i + 6]); - dpdk_prefetch_buffer_by_index (vm, buffers[i + 7]); - - b0 = vlib_get_buffer (vm, buffers[i]); - b1 = vlib_get_buffer (vm, buffers[i + 1]); - b2 = vlib_get_buffer (vm, buffers[i + 2]); - b3 = vlib_get_buffer (vm, buffers[i + 3]); - - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3); - - recycle_or_free (vm, bm, buffers[i], b0); - recycle_or_free (vm, bm, buffers[i + 1], b1); - recycle_or_free (vm, bm, buffers[i + 2], b2); - recycle_or_free (vm, bm, buffers[i + 3], b3); - - i += 4; - } - while (i < n_buffers) - { - b0 = vlib_get_buffer (vm, buffers[i]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - recycle_or_free (vm, bm, buffers[i], b0); - i++; - } - if (vec_len (vm->buffer_announce_list)) + n_left = n_buffers; + bi = buffers; + b = bufp; + vlib_get_buffers (vm, bi, b, n_buffers); + + while (n_left >= 4) { - vlib_buffer_free_list_t *fl; - for (i = 0; i < vec_len (vm->buffer_announce_list); i++) + u32 or_flags; + vlib_buffer_t **p; + + if (n_left < 16) + goto no_prefetch; + + p = b + 12; + dpdk_prefetch_buffer (p[0]); + dpdk_prefetch_buffer (p[1]); + dpdk_prefetch_buffer (p[2]); + dpdk_prefetch_buffer (p[3]); + no_prefetch: + + for (i = 0; i < 4; i++) + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[i]); + + or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags; + + if (or_flags & simple_mask) { - fl = vm->buffer_announce_list[i]; - fl->buffers_added_to_freelist_function (vm, fl); + recycle_or_free (vm, bm, bi[0], b[0]); + recycle_or_free (vm, bm, bi[1], b[1]); + recycle_or_free (vm, bm, bi[2], b[2]); + recycle_or_free (vm, bm, bi[3], b[3]); } - _vec_len (vm->buffer_announce_list) = 0; + else + { + dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0); + dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0); + dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0); + dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0); + } + bi += 4; + b += 4; + n_left -= 4; + } + while (n_left) + { + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + recycle_or_free (vm, bm, bi[0], b[0]); + bi += 1; + b += 1; + n_left -= 1; } - - vec_foreach_index (i, d->mbuf_pending_free_list) - { - int len = vec_len (d->mbuf_pending_free_list[i]); - if (len) - { - rte_mempool_put_bulk (d->mbuf_pending_free_list[i][len - 1]->pool, - (void *) d->mbuf_pending_free_list[i], len); - vec_reset_length (d->mbuf_pending_free_list[i]); - } - } } void @@ -454,109 +390,74 @@ dpdk_packet_template_init (vlib_main_t * vm, vlib_worker_thread_barrier_release (vm); } -static clib_error_t * -scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name) -{ - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - linux_vfio_main_t *lvm = &vfio_main; - const char fn[] = "/dev/vfio/vfio"; - char buff[sizeof (fn)] = { 0 }; - int fd; - u8 *path = format (0, "%v%c", path_name, 0); - - if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn)) - goto done; - - if (strncmp (fn, buff, sizeof (fn))) - goto done; - - fd = atoi ((char *) file_name); - if (fd != lvm->container_fd) - dbm->vfio_container_fd = fd; - -done: - vec_free (path); - return 0; -} - clib_error_t * dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, - struct rte_mempool ** _mp, - vlib_physmem_region_index_t * pri) + struct rte_mempool **_mp, vlib_physmem_region_index_t * pri) { - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; struct rte_mempool *mp; + enum rte_iova_mode iova_mode; vlib_physmem_region_t *pr; dpdk_mempool_private_t priv; clib_error_t *error = 0; - u32 size, obj_size; + size_t min_chunk_size, align; + int map_dma = 1; + u32 size; i32 ret; + uword i; - obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0); - size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0); - - error = - vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa, - VLIB_PHYSMEM_F_HUGETLB | VLIB_PHYSMEM_F_SHARED, - pri); - if (error) - return error; - - pr = vlib_physmem_get_region (vm, pri[0]); - mp = - rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size, - 512, pool_priv_size, numa, 0); + mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size, + 512, pool_priv_size, numa, 0); if (!mp) return clib_error_return (0, "failed to create %s", pool_name); rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); - /* Call the mempool priv initializer */ - priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + - VLIB_BUFFER_DATA_SIZE; - priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; - rte_pktmbuf_pool_init (mp, &priv); + size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21, + &min_chunk_size, &align); - ret = - rte_mempool_populate_iova_tab (mp, pr->mem, pr->page_table, pr->n_pages, - pr->log2_page_size, NULL, NULL); - if (ret != (i32) mp->size) + error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa, + VLIB_PHYSMEM_F_HUGETLB | + VLIB_PHYSMEM_F_SHARED, pri); + if (error) { rte_mempool_free (mp); - return clib_error_return (0, "failed to populate %s", pool_name); + return error; } - _mp[0] = mp; + pr = vlib_physmem_get_region (vm, pri[0]); - /* DPDK currently doesn't provide API to map DMA memory for empty mempool - so we are using this hack, will be nice to have at least API to get - VFIO container FD */ - if (dbm->vfio_container_fd == -1) - foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0); + /* Call the mempool priv initializer */ + priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + + VLIB_BUFFER_DATA_SIZE; + priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; + rte_pktmbuf_pool_init (mp, &priv); - if (dbm->vfio_container_fd != -1) - { - struct vfio_iommu_type1_dma_map dm = { 0 }; - int i, rv = 0; - dm.argsz = sizeof (struct vfio_iommu_type1_dma_map); - dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + if (rte_eth_dev_count_avail () == 0) + map_dma = 0; - /* *INDENT-OFF* */ - vec_foreach_index (i, pr->page_table) + iova_mode = rte_eal_iova_mode (); + for (i = 0; i < pr->n_pages; i++) + { + size_t page_sz = 1ull << pr->log2_page_size; + char *va = ((char *) pr->mem) + i * page_sz; + uword pa = iova_mode == RTE_IOVA_VA ? + pointer_to_uword (va) : pr->page_table[i]; + ret = rte_mempool_populate_iova (mp, va, pa, page_sz, 0, 0); + if (ret < 0) { - dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size); - dm.size = 1 << pr->log2_page_size; - dm.iova = pr->page_table[i]; - if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm))) - break; + rte_mempool_free (mp); + return clib_error_return (0, "failed to populate %s", pool_name); } - /* *INDENT-ON* */ - if (rv != 0 && errno != EINVAL) - clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name); + /* -1 likely means there is no PCI devices assigned to vfio + container or noiommu mode is used so we stop trying */ + if (map_dma && rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz)) + map_dma = 0; } + _mp[0] = mp; + return 0; } @@ -633,8 +534,8 @@ buffer_state_validation_init (vlib_main_t * vm) { void *oldheap; - vlib_buffer_state_heap = mheap_alloc (0, 10 << 20); - + vlib_buffer_state_heap = + mheap_alloc_with_lock (0, 10 << 20, 0 /* locked */ ); oldheap = clib_mem_set_heap (vlib_buffer_state_heap); vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword));