*/
#include <vppinfra/lock.h>
+#include <vlib/dma/dma.h>
#include <vlib/log.h>
#define MEMIF_DEFAULT_SOCKET_FILENAME "memif.sock"
#define MEMIF_DEFAULT_TX_QUEUES 1
#define MEMIF_DEFAULT_BUFFER_SIZE 2048
-#define MEMIF_MAX_M2S_RING (vec_len (vlib_mains) - 1)
-#define MEMIF_MAX_S2M_RING (vec_len (vlib_mains) - 1)
-#define MEMIF_MAX_REGION 255
+#define MEMIF_MAX_M2S_RING 256
+#define MEMIF_MAX_S2M_RING 256
+#define MEMIF_MAX_REGION 256
#define MEMIF_MAX_LOG2_RING_SIZE 14
-#define memif_file_add(a, b) do { \
- *a = clib_file_add (&file_main, b); \
- vlib_log_warn ((&memif_main)->log_class, \
- "clib_file_add fd %d private_data %u idx %u", \
- (b)->file_descriptor, (b)->private_data, *a); \
+#define memif_log_debug(dev, f, ...) do { \
+ memif_if_t *_dev = (memif_if_t *) dev; \
+ if (_dev) \
+ vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, "%U: " f, \
+ format_vnet_hw_if_index_name, vnet_get_main(), \
+ _dev->hw_if_index, ##__VA_ARGS__); \
+ else \
+ vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, f, \
+ ##__VA_ARGS__); \
} while (0)
-#define memif_file_del(a) do { \
- vlib_log_warn ((&memif_main)->log_class, \
- "clib_file_del idx %u",a - file_main.file_pool); \
- clib_file_del (&file_main, a); \
+#define memif_log_warn(dev, f, ...) do { \
+ memif_if_t *_dev = (memif_if_t *) dev; \
+ if (_dev) \
+ vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, "%U: " f, \
+ format_vnet_hw_if_index_name, vnet_get_main(), \
+ _dev->hw_if_index, ##__VA_ARGS__); \
+ else \
+ vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, f, \
+ ##__VA_ARGS__); \
} while (0)
-#define memif_file_del_by_index(a) do { \
- vlib_log_warn ((&memif_main)->log_class, "clib_file_del idx %u", a); \
- clib_file_del_by_index (&file_main, a); \
+#define memif_log_err(dev, f, ...) do { \
+ memif_if_t *_dev = (memif_if_t *) dev; \
+ if (_dev) \
+ vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, "%U: " f, \
+ format_vnet_hw_if_index_name, vnet_get_main(), \
+ _dev->hw_if_index, ##__VA_ARGS__); \
+ else \
+ vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, f, \
+ ##__VA_ARGS__); \
} while (0)
+#define memif_file_add(a, b) \
+ do \
+ { \
+ *a = clib_file_add (&file_main, b); \
+ memif_log_debug (0, "clib_file_add fd %d private_data %u idx %u", \
+ (b)->file_descriptor, (b)->private_data, *a); \
+ } \
+ while (0)
+
+#define memif_file_del(a) \
+ do \
+ { \
+ memif_log_debug (0, "clib_file_del idx %u", a - file_main.file_pool); \
+ clib_file_del (&file_main, a); \
+ } \
+ while (0)
+
+#define memif_file_del_by_index(a) \
+ do \
+ { \
+ memif_log_debug (0, "clib_file_del idx %u", a); \
+ clib_file_del_by_index (&file_main, a); \
+ } \
+ while (0)
+
typedef struct
{
u8 *filename;
int fd;
} memif_msg_fifo_elt_t;
+#define MEMIF_RX_VECTOR_SZ VLIB_FRAME_SIZE
+#define MEMIF_DMA_INFO_SIZE VLIB_FRAME_SIZE
+
+struct memif_dma_info;
+
typedef struct
{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ clib_spinlock_t lockp;
/* ring data */
memif_ring_t *ring;
memif_log2_ring_size_t log2_ring_size;
u16 last_head;
u16 last_tail;
u32 *buffers;
+ u8 buffer_pool_index;
+
+ /* dma data */
+ u16 dma_head;
+ u16 dma_tail;
+ struct memif_dma_info *dma_info;
+ u16 dma_info_head;
+ u16 dma_info_tail;
+ u16 dma_info_size;
+ u8 dma_info_full;
/* interrupts */
int int_fd;
/* queue type */
memif_ring_type_t type;
+ u32 queue_index;
} memif_queue_t;
-#define foreach_memif_if_flag \
- _(0, ADMIN_UP, "admin-up") \
- _(1, IS_SLAVE, "slave") \
- _(2, CONNECTING, "connecting") \
- _(3, CONNECTED, "connected") \
- _(4, DELETING, "deleting") \
- _(5, ZERO_COPY, "zero-copy") \
- _(6, ERROR, "error")
+#define foreach_memif_if_flag \
+ _ (0, ADMIN_UP, "admin-up") \
+ _ (1, IS_SLAVE, "slave") \
+ _ (2, CONNECTING, "connecting") \
+ _ (3, CONNECTED, "connected") \
+ _ (4, DELETING, "deleting") \
+ _ (5, ZERO_COPY, "zero-copy") \
+ _ (6, ERROR, "error") \
+ _ (7, USE_DMA, "use_dma")
typedef enum
{
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
- clib_spinlock_t lockp;
u32 flags;
memif_interface_id_t id;
u32 hw_if_index;
/* disconnect strings */
u8 *local_disc_string;
u8 *remote_disc_string;
+
+ /* dma config index */
+ int dma_input_config;
+ int dma_tx_config;
} memif_if_t;
typedef struct
{
- u32 packet_len;
+ u16 packet_len;
u16 first_buffer_vec_index;
} memif_packet_op_t;
typedef struct
{
+ CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
void *data;
u32 data_len;
i16 buffer_offset;
u16 buffer_vec_index;
} memif_copy_op_t;
-#define MEMIF_RX_VECTOR_SZ VLIB_FRAME_SIZE
+typedef enum
+{
+ MEMIF_DESC_STATUS_OK = 0,
+ MEMIF_DESC_STATUS_ERR_BAD_REGION,
+ MEMIF_DESC_STATUS_ERR_REGION_OVERRUN,
+ MEMIF_DESC_STATUS_ERR_DATA_TOO_BIG,
+ MEMIF_DESC_STATUS_ERR_ZERO_LENGTH
+} __clib_packed memif_desc_status_err_code_t;
+
+typedef union
+{
+ struct
+ {
+ u8 next : 1;
+ u8 err : 1;
+ u8 reserved : 2;
+ memif_desc_status_err_code_t err_code : 4;
+ };
+ u8 as_u8;
+} memif_desc_status_t;
+
+STATIC_ASSERT_SIZEOF (memif_desc_status_t, 1);
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-
+ u16 n_packets;
+ u16 max_desc_len;
+ u32 n_rx_bytes;
+ u8 xor_status;
/* copy vector */
- memif_packet_op_t packet_ops[MEMIF_RX_VECTOR_SZ];
memif_copy_op_t *copy_ops;
u32 *buffers;
+ memif_packet_op_t packet_ops[MEMIF_RX_VECTOR_SZ];
+
+ /* temp storage for compressed descriptors */
+ void **desc_data;
+ u16 *desc_len;
+ memif_desc_status_t *desc_status;
/* buffer template */
vlib_buffer_t buffer_template;
- memif_desc_t desc_template;
} memif_per_thread_data_t;
+typedef struct memif_dma_info
+{
+ /* per thread data */
+ memif_interface_mode_t mode;
+ vlib_node_runtime_t *node;
+ u32 dma_head;
+ u32 dma_tail;
+ u8 finished;
+ memif_per_thread_data_t data;
+} memif_dma_info_t;
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
extern vnet_device_class_t memif_device_class;
extern vlib_node_registration_t memif_input_node;
-enum
+typedef enum
{
MEMIF_PROCESS_EVENT_START = 1,
MEMIF_PROCESS_EVENT_STOP = 2,
+ MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN = 3,
} memif_process_event_t;
typedef struct
u8 *secret;
u8 is_master;
u8 is_zero_copy;
+ u8 use_dma;
memif_interface_mode_t mode:8;
memif_log2_ring_size_t log2_ring_size;
u16 buffer_size;
u32 sw_if_index;
} memif_create_if_args_t;
-int memif_socket_filename_add_del (u8 is_add, u32 sock_id,
- u8 * sock_filename);
-int memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args);
-int memif_delete_if (vlib_main_t * vm, memif_if_t * mif);
+u32 memif_get_unused_socket_id ();
+clib_error_t *memif_socket_filename_add_del (u8 is_add, u32 sock_id,
+ char *sock_filename);
+clib_error_t *memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args);
+clib_error_t *memif_delete_if (vlib_main_t *vm, memif_if_t *mif);
clib_error_t *memif_plugin_api_hookup (vlib_main_t * vm);
+clib_error_t *memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index,
+ u32 flags);
static_always_inline void *
memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot)
clib_error_t *memif_msg_send_disconnect (memif_if_t * mif,
clib_error_t * err);
u8 *format_memif_device_name (u8 * s, va_list * args);
+void memif_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
+void memif_tx_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
/*
* fd.io coding-style-patch-verification: ON