X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fmemif%2Fmemif.c;h=22f3e3dcdf162b202be16404d63877c023bc6964;hb=34c721fb47155135bf2173ca7b9a31aaacfde190;hp=2b25a82f5dd59fdea665e0bd068ce47b35682854;hpb=66593a6a63fe30ed4b8bed96844244d78274e8f2;p=vpp.git diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 2b25a82f5dd..22f3e3dcdf1 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -100,6 +100,8 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) memif_region_t *mr; memif_queue_t *mq; int i; + vlib_main_t *vm = vlib_get_main (); + int with_barrier = 0; if (mif == 0) return; @@ -141,6 +143,12 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) clib_mem_free (mif->sock); } + if (vlib_worker_thread_barrier_held () == 0) + { + with_barrier = 1; + vlib_worker_thread_barrier_sync (vm); + } + /* *INDENT-OFF* */ vec_foreach_index (i, mif->rx_queues) { @@ -198,6 +206,9 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) vec_free (mif->remote_name); vec_free (mif->remote_if_name); clib_fifo_free (mif->msg_queue); + + if (with_barrier) + vlib_worker_thread_barrier_release (vm); } static clib_error_t * @@ -248,6 +259,7 @@ memif_connect (memif_if_t * mif) u32 n_txqs = 0, n_threads = vlib_get_n_threads (); clib_error_t *err = NULL; u8 max_log2_ring_sz = 0; + int with_barrier = 0; memif_log_debug (mif, "connect %u", mif->dev_instance); @@ -278,6 +290,13 @@ memif_connect (memif_if_t * mif) template.read_function = memif_int_fd_read_ready; template.write_function = memif_int_fd_write_ready; + with_barrier = 1; + if (vlib_worker_thread_barrier_held ()) + with_barrier = 0; + + if (with_barrier) + vlib_worker_thread_barrier_sync (vm); + /* *INDENT-OFF* */ vec_foreach_index (i, mif->tx_queues) { @@ -293,6 +312,37 @@ memif_connect (memif_if_t * mif) mq->queue_index = vnet_hw_if_register_tx_queue (vnm, mif->hw_if_index, i); clib_spinlock_init (&mq->lockp); + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } } if (vec_len (mif->tx_queues) > 0) @@ -323,6 +373,37 @@ memif_connect (memif_if_t * mif) qi = vnet_hw_if_register_rx_queue (vnm, mif->hw_if_index, i, VNET_HW_IF_RXQ_THREAD_ANY); mq->queue_index = qi; + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } + if (mq->int_fd > -1) { template.file_descriptor = mq->int_fd; @@ -359,13 +440,6 @@ memif_connect (memif_if_t * mif) if (1 << max_log2_ring_sz > vec_len (mm->per_thread_data[0].desc_data)) { memif_per_thread_data_t *ptd; - int with_barrier = 1; - - if (vlib_worker_thread_barrier_held ()) - with_barrier = 0; - - if (with_barrier) - vlib_worker_thread_barrier_sync (vm); vec_foreach (ptd, mm->per_thread_data) { @@ -376,9 +450,9 @@ memif_connect (memif_if_t * mif) vec_validate_aligned (ptd->desc_status, pow2_mask (max_log2_ring_sz), CLIB_CACHE_LINE_BYTES); } - if (with_barrier) - vlib_worker_thread_barrier_release (vm); } + if (with_barrier) + vlib_worker_thread_barrier_release (vm); mif->flags &= ~MEMIF_IF_FLAG_CONNECTING; mif->flags |= MEMIF_IF_FLAG_CONNECTED; @@ -388,6 +462,8 @@ memif_connect (memif_if_t * mif) return 0; error: + if (with_barrier) + vlib_worker_thread_barrier_release (vm); memif_log_err (mif, "%U", format_clib_error, err); return err; } @@ -638,8 +714,8 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { clib_memset (sock, 0, sizeof(clib_socket_t)); sock->config = (char *) msf->filename; - sock->flags = CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET | - CLIB_SOCKET_F_BLOCKING; + sock->is_seqpacket = 1; + sock->is_blocking = 1; if ((err = clib_socket_init (sock))) { @@ -672,156 +748,155 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) return 0; } -/* *INDENT-OFF* */ VLIB_REGISTER_NODE (memif_process_node,static) = { .function = memif_process, .type = VLIB_NODE_TYPE_PROCESS, .name = "memif-process", }; -/* *INDENT-ON* */ -static int -memif_add_socket_file (u32 sock_id, u8 * socket_filename) +/* + * Returns an unused socket id, and ~0 if it can't find one. + */ +u32 +memif_get_unused_socket_id () { memif_main_t *mm = &memif_main; uword *p; - memif_socket_file_t *msf; + int i, j; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (p) + static u32 seed = 0; + /* limit to 1M tries */ + for (j = 0; j < 1 << 10; j++) { - msf = pool_elt_at_index (mm->socket_files, *p); - if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0) + seed = random_u32 (&seed); + for (i = 0; i < 1 << 10; i++) { - /* Silently accept identical "add". */ - return 0; + /* look around randomly generated id */ + seed += (2 * (i % 2) - 1) * i; + if (seed == (u32) ~0) + continue; + p = hash_get (mm->socket_file_index_by_sock_id, seed); + if (!p) + return seed; } - - /* But don't allow a direct add of a different filename. */ - return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; } - pool_get (mm->socket_files, msf); - clib_memset (msf, 0, sizeof (memif_socket_file_t)); - - msf->filename = socket_filename; - msf->socket_id = sock_id; - - hash_set (mm->socket_file_index_by_sock_id, sock_id, - msf - mm->socket_files); - - return 0; + return ~0; } -static int -memif_delete_socket_file (u32 sock_id) +clib_error_t * +memif_socket_filename_add_del (u8 is_add, u32 sock_id, char *sock_filename) { memif_main_t *mm = &memif_main; uword *p; memif_socket_file_t *msf; + clib_error_t *err = 0; + char *dir = 0, *tmp; + u32 idx = 0; + u8 *name = 0; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (!p) - { - /* Don't delete non-existent entries. */ - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* allow adding socket id 0 */ + if (sock_id == 0 && is_add == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, "cannot delete socket id 0"); - msf = pool_elt_at_index (mm->socket_files, *p); - if (msf->ref_cnt > 0) - { - return VNET_API_ERROR_UNEXPECTED_INTF_STATE; - } + if (sock_id == ~0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socked id is not specified"); - vec_free (msf->filename); - pool_put (mm->socket_files, msf); + if (is_add == 0) + { + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (!p) + /* Don't delete non-existent entries. */ + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket file with id %u does not exist", sock_id); - hash_unset (mm->socket_file_index_by_sock_id, sock_id); + msf = pool_elt_at_index (mm->socket_files, *p); + if (msf->ref_cnt > 0) + return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE, + "socket file '%s' is in use", msf->filename); - return 0; -} + vec_free (msf->filename); + pool_put (mm->socket_files, msf); -int -memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename) -{ - char *dir = 0, *tmp; - u32 idx = 0; + hash_unset (mm->socket_file_index_by_sock_id, sock_id); - /* allow adding socket id 0 */ - if ((sock_id == 0 && is_add == 0) || sock_id == ~0) - { - return VNET_API_ERROR_INVALID_ARGUMENT; + return 0; } - if (is_add == 0) + if (sock_filename == 0 || sock_filename[0] == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket filename not specified"); + + if (clib_socket_prefix_is_valid (sock_filename)) { - return memif_delete_socket_file (sock_id); + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename == 0 || sock_filename[0] == 0) + else if (sock_filename[0] == '/') { - return VNET_API_ERROR_INVALID_ARGUMENT; + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename[0] != '/') + else { - clib_error_t *error; - /* copy runtime dir path */ vec_add (dir, vlib_unix_get_runtime_dir (), strlen (vlib_unix_get_runtime_dir ())); vec_add1 (dir, '/'); /* if sock_filename contains dirs, add them to path */ - tmp = strrchr ((char *) sock_filename, '/'); + tmp = strrchr (sock_filename, '/'); if (tmp) { - idx = tmp - (char *) sock_filename; + idx = tmp - sock_filename; vec_add (dir, sock_filename, idx); } vec_add1 (dir, '\0'); /* create socket dir */ - error = vlib_unix_recursive_mkdir (dir); - if (error) + if ((err = vlib_unix_recursive_mkdir (dir))) { - clib_error_free (error); - return VNET_API_ERROR_SYSCALL_ERROR_1; + clib_error_free (err); + err = vnet_error (VNET_ERR_SYSCALL_ERROR_1, + "unable to create socket dir"); + goto done; } - sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), - sock_filename, 0); + name = + format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), sock_filename, 0); } - else - { - sock_filename = vec_dup (sock_filename); - /* check if directory exists */ - tmp = strrchr ((char *) sock_filename, '/'); - if (tmp) + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (p) + { + msf = pool_elt_at_index (mm->socket_files, *p); + if (strcmp ((char *) msf->filename, (char *) name) == 0) { - idx = tmp - (char *) sock_filename; - vec_add (dir, sock_filename, idx); - vec_add1 (dir, '\0'); + /* Silently accept identical "add". */ + goto done; } - /* check dir existance and access rights for effective user/group IDs */ - if ((dir == NULL) - || - (faccessat ( /* ignored */ -1, dir, F_OK | R_OK | W_OK, AT_EACCESS) - < 0)) - { - vec_free (dir); - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* But don't allow a direct add of a different filename. */ + err = vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS, "entry already exists"); + goto done; } - vec_free (dir); - return memif_add_socket_file (sock_id, sock_filename); + pool_get (mm->socket_files, msf); + clib_memset (msf, 0, sizeof (memif_socket_file_t)); + + msf->filename = name; + msf->socket_id = sock_id; + name = 0; + + hash_set (mm->socket_file_index_by_sock_id, sock_id, msf - mm->socket_files); + +done: + vec_free (name); + vec_free (dir); + return err; } -int -memif_delete_if (vlib_main_t * vm, memif_if_t * mif) +clib_error_t * +memif_delete_if (vlib_main_t *vm, memif_if_t *mif) { vnet_main_t *vnm = vnet_get_main (); memif_main_t *mm = &memif_main; @@ -900,8 +975,18 @@ VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = { }; /* *INDENT-ON* */ -int -memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) +static void +memif_prepare_dma_args (vlib_dma_config_t *args) +{ + args->max_batches = 256; + args->max_transfer_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; + args->barrier_before_last = 1; + args->sw_fallback = 1; + args->callback_fn = NULL; +} + +clib_error_t * +memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args) { memif_main_t *mm = &memif_main; vlib_thread_main_t *tm = vlib_get_thread_main (); @@ -909,16 +994,14 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) vnet_eth_interface_registration_t eir = {}; memif_if_t *mif = 0; vnet_sw_interface_t *sw; - clib_error_t *error = 0; - int ret = 0; uword *p; memif_socket_file_t *msf = 0; - int rv = 0; + clib_error_t *err = 0; p = hash_get (mm->socket_file_index_by_sock_id, args->socket_id); if (p == 0) { - rv = VNET_API_ERROR_INVALID_ARGUMENT; + err = vnet_error (VNET_ERR_INVALID_ARGUMENT, "unknown socket id"); goto done; } @@ -929,14 +1012,17 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) { if ((!msf->is_listener != !args->is_master)) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = + vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "socket file cannot be used by both master and slave"); goto done; } p = mhash_get (&msf->dev_instance_by_id, &args->id); if (p) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "interface already exists"); goto done; } } @@ -944,25 +1030,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* Create new socket file */ if (msf->ref_cnt == 0) { - struct stat file_stat; - - /* If we are creating listener make sure file doesn't exist or if it - * exists thn delete it if it is old socket file */ - if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0)) - { - if (S_ISSOCK (file_stat.st_mode)) - { - unlink ((char *) msf->filename); - } - else - { - error = clib_error_return (0, "File exists for %s", - msf->filename); - rv = VNET_API_ERROR_VALUE_EXIST; - goto done; - } - } - mhash_init (&msf->dev_instance_by_id, sizeof (uword), sizeof (memif_interface_id_t)); msf->dev_instance_by_fd = hash_create (0, sizeof (uword)); @@ -988,8 +1055,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) bt->total_length_not_including_first_buffer = 0; vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0; - /* initially prealloc copy_ops so we can use - _vec_len instead of vec_elen */ vec_validate_aligned (ptd->copy_ops, 0, CLIB_CACHE_LINE_BYTES); vec_reset_length (ptd->copy_ops); vec_validate_aligned (ptd->buffers, 0, CLIB_CACHE_LINE_BYTES); @@ -1007,6 +1072,20 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) if (args->secret) mif->secret = vec_dup (args->secret); + /* register dma config if enabled */ + if (args->use_dma) + { + vlib_dma_config_t dma_args; + bzero (&dma_args, sizeof (dma_args)); + memif_prepare_dma_args (&dma_args); + + dma_args.max_transfers = 1 << args->log2_ring_size; + dma_args.callback_fn = memif_dma_completion_cb; + mif->dma_input_config = vlib_dma_config_add (vm, &dma_args); + dma_args.callback_fn = memif_tx_dma_completion_cb; + mif->dma_tx_config = vlib_dma_config_add (vm, &dma_args); + } + if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET) { @@ -1038,11 +1117,9 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->dev_instance); } else - error = clib_error_return (0, "unsupported interface mode"); - - if (error) { - ret = VNET_API_ERROR_SYSCALL_ERROR_2; + err = + vnet_error (VNET_ERR_SYSCALL_ERROR_2, "unsupported interface mode"); goto error; } @@ -1061,7 +1138,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* If this is new one, start listening */ if (msf->is_listener && msf->ref_cnt == 0) { - struct stat file_stat; clib_socket_t *s = clib_mem_alloc (sizeof (clib_socket_t)); ASSERT (msf->sock == 0); @@ -1069,19 +1145,15 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) clib_memset (s, 0, sizeof (clib_socket_t)); s->config = (char *) msf->filename; - s->flags = CLIB_SOCKET_F_IS_SERVER | - CLIB_SOCKET_F_ALLOW_GROUP_WRITE | - CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED; - - if ((error = clib_socket_init (s))) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_4; - goto error; - } + s->local_only = 1; + s->is_server = 1; + s->allow_group_write = 1; + s->is_seqpacket = 1; + s->passcred = 1; - if (stat ((char *) msf->filename, &file_stat) == -1) + if ((err = clib_socket_init (s))) { - ret = VNET_API_ERROR_SYSCALL_ERROR_8; + err->code = VNET_ERR_SYSCALL_ERROR_4; goto error; } @@ -1102,6 +1174,9 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->flags |= MEMIF_IF_FLAG_ZERO_COPY; } + if (args->use_dma) + mif->flags |= MEMIF_IF_FLAG_USE_DMA; + vnet_hw_if_set_caps (vnm, mif->hw_if_index, VNET_HW_IF_CAP_INT_MODE); vnet_hw_if_set_input_node (vnm, mif->hw_if_index, memif_input_node.index); mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0); @@ -1115,15 +1190,12 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) error: memif_delete_if (vm, mif); - if (error) - { - memif_log_err (mif, "%U", format_clib_error, error); - clib_error_free (error); - } - return ret; + if (err) + memif_log_err (mif, "%U", format_clib_error, err); + return err; done: - return rv; + return err; } clib_error_t * @@ -1169,9 +1241,7 @@ memif_init (vlib_main_t * vm) * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the * default run-time directory. */ - memif_socket_filename_add_del (1, 0, (u8 *) MEMIF_DEFAULT_SOCKET_FILENAME); - - return 0; + return memif_socket_filename_add_del (1, 0, MEMIF_DEFAULT_SOCKET_FILENAME); } VLIB_INIT_FUNCTION (memif_init);