2 *------------------------------------------------------------------
3 * Copyright (c) 2017 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vppinfra/lock.h>
19 #include <vlib/dma/dma.h>
22 #define MEMIF_DEFAULT_SOCKET_FILENAME "memif.sock"
23 #define MEMIF_DEFAULT_RING_SIZE 1024
24 #define MEMIF_DEFAULT_RX_QUEUES 1
25 #define MEMIF_DEFAULT_TX_QUEUES 1
26 #define MEMIF_DEFAULT_BUFFER_SIZE 2048
28 #define MEMIF_MAX_M2S_RING 256
29 #define MEMIF_MAX_S2M_RING 256
30 #define MEMIF_MAX_REGION 256
31 #define MEMIF_MAX_LOG2_RING_SIZE 14
34 #define memif_log_debug(dev, f, ...) do { \
35 memif_if_t *_dev = (memif_if_t *) dev; \
37 vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, "%U: " f, \
38 format_vnet_hw_if_index_name, vnet_get_main(), \
39 _dev->hw_if_index, ##__VA_ARGS__); \
41 vlib_log(VLIB_LOG_LEVEL_DEBUG, memif_main.log_class, f, \
45 #define memif_log_warn(dev, f, ...) do { \
46 memif_if_t *_dev = (memif_if_t *) dev; \
48 vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, "%U: " f, \
49 format_vnet_hw_if_index_name, vnet_get_main(), \
50 _dev->hw_if_index, ##__VA_ARGS__); \
52 vlib_log(VLIB_LOG_LEVEL_WARNING, memif_main.log_class, f, \
56 #define memif_log_err(dev, f, ...) do { \
57 memif_if_t *_dev = (memif_if_t *) dev; \
59 vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, "%U: " f, \
60 format_vnet_hw_if_index_name, vnet_get_main(), \
61 _dev->hw_if_index, ##__VA_ARGS__); \
63 vlib_log(VLIB_LOG_LEVEL_ERR, memif_main.log_class, f, \
67 #define memif_file_add(a, b) \
70 *a = clib_file_add (&file_main, b); \
71 memif_log_debug (0, "clib_file_add fd %d private_data %u idx %u", \
72 (b)->file_descriptor, (b)->private_data, *a); \
76 #define memif_file_del(a) \
79 memif_log_debug (0, "clib_file_del idx %u", a - file_main.file_pool); \
80 clib_file_del (&file_main, a); \
84 #define memif_file_del_by_index(a) \
87 memif_log_debug (0, "clib_file_del idx %u", a); \
88 clib_file_del_by_index (&file_main, a); \
97 clib_socket_t **pending_clients;
101 /* hash of all registered id */
102 mhash_t dev_instance_by_id;
104 /* hash of all registered fds */
105 uword *dev_instance_by_fd;
106 } memif_socket_file_t;
110 /* Required for vec_validate_aligned */
111 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
113 memif_region_size_t region_size;
122 } memif_msg_fifo_elt_t;
124 #define MEMIF_RX_VECTOR_SZ VLIB_FRAME_SIZE
125 #define MEMIF_DMA_INFO_SIZE VLIB_FRAME_SIZE
127 struct memif_dma_info;
131 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
132 clib_spinlock_t lockp;
135 memif_log2_ring_size_t log2_ring_size;
136 memif_region_index_t region;
137 memif_region_offset_t offset;
142 u8 buffer_pool_index;
147 struct memif_dma_info *dma_info;
155 uword int_clib_file_index;
159 memif_ring_type_t type;
163 #define foreach_memif_if_flag \
164 _ (0, ADMIN_UP, "admin-up") \
165 _ (1, IS_SLAVE, "slave") \
166 _ (2, CONNECTING, "connecting") \
167 _ (3, CONNECTED, "connected") \
168 _ (4, DELETING, "deleting") \
169 _ (5, ZERO_COPY, "zero-copy") \
170 _ (6, ERROR, "error") \
171 _ (7, USE_DMA, "use_dma")
175 #define _(a, b, c) MEMIF_IF_FLAG_##b = (1 << a),
176 foreach_memif_if_flag
182 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
184 memif_interface_id_t id;
188 memif_interface_mode_t mode:8;
190 u32 per_interface_next_index;
192 /* socket connection */
194 uword socket_file_index;
195 memif_msg_fifo_elt_t *msg_queue;
198 memif_region_t *regions;
200 memif_queue_t *rx_queues;
201 memif_queue_t *tx_queues;
209 memif_log2_ring_size_t log2_ring_size;
217 memif_log2_ring_size_t log2_ring_size;
223 /* disconnect strings */
224 u8 *local_disc_string;
225 u8 *remote_disc_string;
227 /* dma config index */
228 int dma_input_config;
235 u16 first_buffer_vec_index;
240 CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
244 u16 buffer_vec_index;
249 MEMIF_DESC_STATUS_OK = 0,
250 MEMIF_DESC_STATUS_ERR_BAD_REGION,
251 MEMIF_DESC_STATUS_ERR_REGION_OVERRUN,
252 MEMIF_DESC_STATUS_ERR_DATA_TOO_BIG,
253 MEMIF_DESC_STATUS_ERR_ZERO_LENGTH
254 } __clib_packed memif_desc_status_err_code_t;
263 memif_desc_status_err_code_t err_code : 4;
266 } memif_desc_status_t;
268 STATIC_ASSERT_SIZEOF (memif_desc_status_t, 1);
272 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
278 memif_copy_op_t *copy_ops;
280 memif_packet_op_t packet_ops[MEMIF_RX_VECTOR_SZ];
282 /* temp storage for compressed descriptors */
285 memif_desc_status_t *desc_status;
287 /* buffer template */
288 vlib_buffer_t buffer_template;
289 } memif_per_thread_data_t;
291 typedef struct memif_dma_info
293 /* per thread data */
294 memif_interface_mode_t mode;
295 vlib_node_runtime_t *node;
299 memif_per_thread_data_t data;
304 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
306 /** API message ID base */
309 /* pool of all memory interfaces */
310 memif_if_t *interfaces;
312 /* pool of all unix socket files */
313 memif_socket_file_t *socket_files;
314 uword *socket_file_index_by_sock_id; /* map user socket id to pool idx */
316 /* per thread data */
317 memif_per_thread_data_t *per_thread_data;
319 vlib_log_class_t log_class;
323 extern memif_main_t memif_main;
324 extern vnet_device_class_t memif_device_class;
325 extern vlib_node_registration_t memif_input_node;
329 MEMIF_PROCESS_EVENT_START = 1,
330 MEMIF_PROCESS_EVENT_STOP = 2,
331 MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN = 3,
332 } memif_process_event_t;
336 memif_interface_id_t id;
342 memif_interface_mode_t mode:8;
343 memif_log2_ring_size_t log2_ring_size;
352 } memif_create_if_args_t;
354 u32 memif_get_unused_socket_id ();
355 clib_error_t *memif_socket_filename_add_del (u8 is_add, u32 sock_id,
356 char *sock_filename);
357 clib_error_t *memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args);
358 clib_error_t *memif_delete_if (vlib_main_t *vm, memif_if_t *mif);
359 clib_error_t *memif_plugin_api_hookup (vlib_main_t * vm);
360 clib_error_t *memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index,
363 static_always_inline void *
364 memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot)
366 u16 region = ring->desc[slot].region;
367 return mif->regions[region].shm + ring->desc[slot].offset;
371 clib_error_t *memif_init_regions_and_queues (memif_if_t * mif);
372 clib_error_t *memif_connect (memif_if_t * mif);
373 void memif_disconnect (memif_if_t * mif, clib_error_t * err);
376 void memif_socket_close (clib_socket_t ** sock);
377 clib_error_t *memif_conn_fd_accept_ready (clib_file_t * uf);
378 clib_error_t *memif_master_conn_fd_read_ready (clib_file_t * uf);
379 clib_error_t *memif_slave_conn_fd_read_ready (clib_file_t * uf);
380 clib_error_t *memif_master_conn_fd_write_ready (clib_file_t * uf);
381 clib_error_t *memif_slave_conn_fd_write_ready (clib_file_t * uf);
382 clib_error_t *memif_master_conn_fd_error (clib_file_t * uf);
383 clib_error_t *memif_slave_conn_fd_error (clib_file_t * uf);
384 clib_error_t *memif_msg_send_disconnect (memif_if_t * mif,
386 u8 *format_memif_device_name (u8 * s, va_list * args);
387 void memif_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
388 void memif_tx_dma_completion_cb (vlib_main_t *vm, vlib_dma_batch_t *b);
391 * fd.io coding-style-patch-verification: ON
394 * eval: (c-set-style "gnu")