* Allocate/free network buffers.
*/
+#include <unistd.h>
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+
#include <rte_config.h>
#include <rte_common.h>
#include <rte_version.h>
#include <vlib/vlib.h>
+#include <vlib/unix/unix.h>
+#include <vlib/pci/pci.h>
+#include <vlib/linux/vfio.h>
#include <vnet/vnet.h>
#include <dpdk/device/dpdk.h>
#include <dpdk/device/dpdk_priv.h>
-
STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
typedef struct
{
+ int vfio_container_fd;
dpdk_buffer_per_thread_data *ptd;
} dpdk_buffer_main_t;
/* Add buffer free list. */
static void
-dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
+dpdk_buffer_delete_free_list (vlib_main_t * vm,
+ vlib_buffer_free_list_index_t free_list_index)
{
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_free_list_t *f;
- u32 merge_index;
+ vlib_buffer_free_list_index_t merge_index;
int i;
ASSERT (vlib_get_thread_index () == 0);
f = vlib_buffer_get_free_list (vm, free_list_index);
merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
- if (merge_index != ~0 && merge_index != free_list_index)
+ if (merge_index != (vlib_buffer_free_list_index_t) ~ 0 &&
+ merge_index != free_list_index)
{
vlib_buffer_merge_free_lists (pool_elt_at_index
(bm->buffer_free_list_pool, merge_index),
#endif
/* Make sure free list has at least given number of free buffers. */
-static uword
-fill_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * fl, uword min_free_buffers)
+uword
+CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm,
+ vlib_buffer_free_list_t * fl,
+ uword min_free_buffers)
{
dpdk_main_t *dm = &dpdk_main;
vlib_buffer_t *b0, *b1, *b2, *b3;
return n;
}
-static u32
-alloc_from_free_list (vlib_main_t * vm,
- vlib_buffer_free_list_t * free_list,
- u32 * alloc_buffers, u32 n_alloc_buffers)
-{
- u32 *dst, *src;
- uword len, n_filled;
-
- dst = alloc_buffers;
-
- n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
- if (n_filled == 0)
- return 0;
-
- len = vec_len (free_list->buffers);
- ASSERT (len >= n_alloc_buffers);
-
- src = free_list->buffers + len - n_alloc_buffers;
- clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
- _vec_len (free_list->buffers) -= n_alloc_buffers;
-
- return n_alloc_buffers;
-}
-
-/* Allocate a given number of buffers into given array.
- Returns number actually allocated which will be either zero or
- number requested. */
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
- u32 n_buffers)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
-
- return alloc_from_free_list
- (vm,
- pool_elt_at_index (bm->buffer_free_list_pool,
- VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
- buffers, n_buffers);
-}
-
-
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
- u32 * buffers,
- u32 n_buffers,
- u32 free_list_index)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_buffer_free_list_t *f;
- f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
- return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
static_always_inline void
dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
{
{
vlib_buffer_free_list_t *fl;
u32 thread_index = vlib_get_thread_index ();
- u32 fi;
+ vlib_buffer_free_list_index_t fi;
fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
/* The only current use of this callback: multicast recycle */
while (i < n_buffers)
{
b0 = vlib_get_buffer (vm, buffers[i]);
- VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b);
+ VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
recycle_or_free (vm, bm, buffers[i], b0);
i++;
}
vlib_worker_thread_barrier_release (vm);
}
+static clib_error_t *
+scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name)
+{
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
+ linux_vfio_main_t *lvm = &vfio_main;
+ const char fn[] = "/dev/vfio/vfio";
+ char buff[sizeof (fn)] = { 0 };
+ int fd;
+ u8 *path = format (0, "%v%c", path_name, 0);
+
+ if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn))
+ goto done;
+
+ if (strncmp (fn, buff, sizeof (fn)))
+ goto done;
+
+ fd = atoi ((char *) file_name);
+ if (fd != lvm->container_fd)
+ dbm->vfio_container_fd = fd;
+
+done:
+ vec_free (path);
+ return 0;
+}
+
clib_error_t *
dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size,
u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa,
- struct rte_mempool **_mp, vlib_physmem_region_index_t * pri)
+ struct rte_mempool ** _mp,
+ vlib_physmem_region_index_t * pri)
{
+ dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
struct rte_mempool *mp;
vlib_physmem_region_t *pr;
clib_error_t *error = 0;
i32 ret;
obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0);
-#if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0)
- size = rte_mempool_xmem_size (num_elts, obj_size, 21);
-#else
size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0);
-#endif
error =
- vlib_physmem_region_alloc (vm, (i8 *) pool_name, size, numa, 0, pri);
+ vlib_physmem_region_alloc (vm, (i8 *) pool_name, size, numa,
+ VLIB_PHYSMEM_F_HUGETLB | VLIB_PHYSMEM_F_SHARED,
+ pri);
if (error)
return error;
rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL);
-#if RTE_VERSION < RTE_VERSION_NUM(17, 11, 0, 0)
- ret =
- rte_mempool_populate_phys_tab (mp, pr->mem, pr->page_table, pr->n_pages,
- pr->log2_page_size, NULL, NULL);
-#else
ret =
rte_mempool_populate_iova_tab (mp, pr->mem, pr->page_table, pr->n_pages,
pr->log2_page_size, NULL, NULL);
-#endif
if (ret != (i32) mp->size)
{
rte_mempool_free (mp);
_mp[0] = mp;
+ /* DPDK currently doesn't provide API to map DMA memory for empty mempool
+ so we are using this hack, will be nice to have at least API to get
+ VFIO container FD */
+ if (dbm->vfio_container_fd == -1)
+ foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0);
+
+ if (dbm->vfio_container_fd != -1)
+ {
+ struct vfio_iommu_type1_dma_map dm = { 0 };
+ int i, rv = 0;
+ dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
+ dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
+
+ /* *INDENT-OFF* */
+ vec_foreach_index (i, pr->page_table)
+ {
+ dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
+ dm.size = 1 << pr->log2_page_size;
+ dm.iova = pr->page_table[i];
+ if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm)))
+ break;
+ }
+ /* *INDENT-ON* */
+ if (rv != 0 && rv != EINVAL)
+ clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name);
+ }
+
return 0;
}
{
dpdk_buffer_main_t *dbm = &dpdk_buffer_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
+
vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1,
CLIB_CACHE_LINE_BYTES);
+
+ dbm->vfio_container_fd = -1;
+
return 0;
}
/* *INDENT-OFF* */
VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
- .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
- .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
+ .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
.vlib_buffer_free_cb = &dpdk_buffer_free,
.vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
.vlib_packet_template_init_cb = &dpdk_packet_template_init,
/* *INDENT-ON* */
#if __x86_64__
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx512;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
- dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512;
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2;
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512;
dpdk_input_multiarch_select (void)
{
vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
- if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+ if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx512;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512;
}
- else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+ else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
{
- cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
- cb->vlib_buffer_alloc_from_free_list_cb =
- dpdk_buffer_alloc_from_free_list_avx2;
+ cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2;
cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2;
cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2;
}