X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fmemif%2Fmemif.c;h=22f3e3dcdf162b202be16404d63877c023bc6964;hb=34c721fb47155135bf2173ca7b9a31aaacfde190;hp=763572c2f0ed5a27b1d07b48686a179de88626f5;hpb=b7b929931a07fbb27b43d5cd105f366c3e29807e;p=vpp.git diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 763572c2f0e..22f3e3dcdf1 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -33,9 +33,10 @@ #include #include -#include #include #include +#include +#include #include #include #include @@ -49,6 +50,14 @@ memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) return 0; } +static clib_error_t * +memif_eth_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hi, + u32 flags) +{ + /* nothing for now */ + return 0; +} + static void memif_queue_intfd_close (memif_queue_t * mq) { @@ -65,6 +74,24 @@ memif_queue_intfd_close (memif_queue_t * mq) } } +static void +memif_disconnect_free_zc_queue_buffer (memif_queue_t * mq, u8 is_rx) +{ + vlib_main_t *vm = vlib_get_main (); + u16 ring_size, n_slots, mask, start; + + ring_size = 1 << mq->log2_ring_size; + mask = ring_size - 1; + n_slots = mq->ring->head - mq->last_tail; + start = mq->last_tail & mask; + if (is_rx) + vlib_buffer_free_from_ring (vm, mq->buffers, start, ring_size, n_slots); + else + vlib_buffer_free_from_ring_no_next (vm, mq->buffers, start, ring_size, + n_slots); + vec_free (mq->buffers); +} + void memif_disconnect (memif_if_t * mif, clib_error_t * err) { @@ -73,6 +100,8 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) memif_region_t *mr; memif_queue_t *mq; int i; + vlib_main_t *vm = vlib_get_main (); + int with_barrier = 0; if (mif == 0) return; @@ -114,21 +143,43 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) clib_mem_free (mif->sock); } + if (vlib_worker_thread_barrier_held () == 0) + { + with_barrier = 1; + vlib_worker_thread_barrier_sync (vm); + } + /* *INDENT-OFF* */ vec_foreach_index (i, mif->rx_queues) { mq = vec_elt_at_index (mif->rx_queues, i); if (mq->ring) { - int rv; - rv = vnet_hw_interface_unassign_rx_thread (vnm, mif->hw_if_index, i); - if (rv) - memif_log_warn (mif, - "Unable to unassign interface %d, queue %d: rc=%d", - mif->hw_if_index, i, rv); + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + { + memif_disconnect_free_zc_queue_buffer(mq, 1); + } mq->ring = 0; } } + vnet_hw_if_unregister_all_rx_queues (vnm, mif->hw_if_index); + + /* *INDENT-OFF* */ + vec_foreach_index (i, mif->tx_queues) + { + mq = vec_elt_at_index (mif->tx_queues, i); + if (mq->ring) + { + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + { + memif_disconnect_free_zc_queue_buffer(mq, 0); + } + clib_spinlock_free (&mq->lockp); + } + mq->ring = 0; + } + vnet_hw_if_unregister_all_tx_queues (vnm, mif->hw_if_index); + vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index); /* free tx and rx queues */ vec_foreach (mq, mif->rx_queues) @@ -155,6 +206,20 @@ memif_disconnect (memif_if_t * mif, clib_error_t * err) vec_free (mif->remote_name); vec_free (mif->remote_if_name); clib_fifo_free (mif->msg_queue); + + if (with_barrier) + vlib_worker_thread_barrier_release (vm); +} + +static clib_error_t * +memif_int_fd_write_ready (clib_file_t * uf) +{ + memif_main_t *mm = &memif_main; + u16 qid = uf->private_data & 0xFFFF; + memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16); + + memif_log_warn (mif, "unexpected EPOLLOUT on RX for queue %u", qid); + return 0; } static clib_error_t * @@ -171,11 +236,11 @@ memif_int_fd_read_ready (clib_file_t * uf) size = read (uf->file_descriptor, &b, sizeof (b)); if (size < 0) { - memif_log_debug (mif, "Failed to read form socket"); + memif_log_debug (mif, "Failed to read from socket"); return 0; } - vnet_device_input_set_interrupt_pending (vnm, mif->hw_if_index, qid); + vnet_hw_if_rx_queue_set_int_pending (vnm, mq->queue_index); mq->int_count++; return 0; @@ -185,11 +250,16 @@ memif_int_fd_read_ready (clib_file_t * uf) clib_error_t * memif_connect (memif_if_t * mif) { + memif_main_t *mm = &memif_main; + vlib_main_t *vm = vlib_get_main (); vnet_main_t *vnm = vnet_get_main (); clib_file_t template = { 0 }; memif_region_t *mr; - int i; + int i, j; + u32 n_txqs = 0, n_threads = vlib_get_n_threads (); clib_error_t *err = NULL; + u8 max_log2_ring_sz = 0; + int with_barrier = 0; memif_log_debug (mif, "connect %u", mif->dev_instance); @@ -218,11 +288,20 @@ memif_connect (memif_if_t * mif) /* *INDENT-ON* */ template.read_function = memif_int_fd_read_ready; + template.write_function = memif_int_fd_write_ready; + + with_barrier = 1; + if (vlib_worker_thread_barrier_held ()) + with_barrier = 0; + + if (with_barrier) + vlib_worker_thread_barrier_sync (vm); /* *INDENT-OFF* */ vec_foreach_index (i, mif->tx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i); + max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size); mq->ring = mif->regions[mq->region].shm + mq->offset; if (mq->ring->cookie != MEMIF_COOKIE) @@ -230,19 +309,100 @@ memif_connect (memif_if_t * mif) err = clib_error_return (0, "wrong cookie on tx ring %u", i); goto error; } + mq->queue_index = + vnet_hw_if_register_tx_queue (vnm, mif->hw_if_index, i); + clib_spinlock_init (&mq->lockp); + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } + } + + if (vec_len (mif->tx_queues) > 0) + { + n_txqs = vec_len (mif->tx_queues); + for (j = 0; j < n_threads; j++) + { + u32 qi = mif->tx_queues[j % n_txqs].queue_index; + vnet_hw_if_tx_queue_assign_thread (vnm, qi, j); + } } vec_foreach_index (i, mif->rx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i); + u32 ti; + u32 qi; int rv; + max_log2_ring_sz = clib_max (max_log2_ring_sz, mq->log2_ring_size); + mq->ring = mif->regions[mq->region].shm + mq->offset; if (mq->ring->cookie != MEMIF_COOKIE) { err = clib_error_return (0, "wrong cookie on tx ring %u", i); goto error; } + qi = vnet_hw_if_register_rx_queue (vnm, mif->hw_if_index, i, + VNET_HW_IF_RXQ_THREAD_ANY); + mq->queue_index = qi; + + if (mif->flags & MEMIF_IF_FLAG_USE_DMA) + { + memif_dma_info_t *dma_info; + mq->dma_head = 0; + mq->dma_tail = 0; + mq->dma_info_head = 0; + mq->dma_info_tail = 0; + mq->dma_info_size = MEMIF_DMA_INFO_SIZE; + vec_validate_aligned (mq->dma_info, MEMIF_DMA_INFO_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_foreach (dma_info, mq->dma_info) + { + vec_validate_aligned (dma_info->data.desc_data, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_len, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.desc_status, + pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dma_info->data.copy_ops, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.copy_ops); + vec_validate_aligned (dma_info->data.buffers, 0, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (dma_info->data.buffers); + } + } if (mq->int_fd > -1) { @@ -252,27 +412,48 @@ memif_connect (memif_if_t * mif) format_memif_device_name, mif->dev_instance, i); memif_file_add (&mq->int_clib_file_index, &template); + vnet_hw_if_set_rx_queue_file_index (vnm, qi, + mq->int_clib_file_index); } - vnet_hw_interface_assign_rx_thread (vnm, mif->hw_if_index, i, ~0); - rv = vnet_hw_interface_set_rx_mode (vnm, mif->hw_if_index, i, - VNET_HW_INTERFACE_RX_MODE_DEFAULT); + ti = vnet_hw_if_get_rx_queue_thread_index (vnm, qi); + mq->buffer_pool_index = vlib_buffer_pool_get_default_for_numa ( + vm, vlib_get_main_by_index (ti)->numa_node); + rv = vnet_hw_if_set_rx_queue_mode (vnm, qi, VNET_HW_IF_RX_MODE_DEFAULT); + vnet_hw_if_update_runtime_data (vnm, mif->hw_if_index); + if (rv) memif_log_err (mif, "Warning: unable to set rx mode for interface %d queue %d: " "rc=%d", mif->hw_if_index, i, rv); else { - vnet_hw_interface_rx_mode rxmode; - vnet_hw_interface_get_rx_mode (vnm, mif->hw_if_index, i, &rxmode); + vnet_hw_if_rx_mode rxmode = vnet_hw_if_get_rx_queue_mode (vnm, qi); - if (rxmode == VNET_HW_INTERFACE_RX_MODE_POLLING) + if (rxmode == VNET_HW_IF_RX_MODE_POLLING) mq->ring->flags |= MEMIF_RING_FLAG_MASK_INT; else - vnet_device_input_set_interrupt_pending (vnm, mif->hw_if_index, i); + vnet_hw_if_rx_queue_set_int_pending (vnm, qi); } } /* *INDENT-ON* */ + if (1 << max_log2_ring_sz > vec_len (mm->per_thread_data[0].desc_data)) + { + memif_per_thread_data_t *ptd; + + vec_foreach (ptd, mm->per_thread_data) + { + vec_validate_aligned (ptd->desc_data, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (ptd->desc_len, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (ptd->desc_status, pow2_mask (max_log2_ring_sz), + CLIB_CACHE_LINE_BYTES); + } + } + if (with_barrier) + vlib_worker_thread_barrier_release (vm); + mif->flags &= ~MEMIF_IF_FLAG_CONNECTING; mif->flags |= MEMIF_IF_FLAG_CONNECTED; @@ -281,6 +462,8 @@ memif_connect (memif_if_t * mif) return 0; error: + if (with_barrier) + vlib_worker_thread_barrier_release (vm); memif_log_err (mif, "%U", format_clib_error, err); return err; } @@ -303,11 +486,11 @@ clib_error_t * memif_init_regions_and_queues (memif_if_t * mif) { vlib_main_t *vm = vlib_get_main (); + memif_socket_file_t *msf; memif_ring_t *ring = NULL; - int i, j; + int fd, i, j; u64 buffer_offset; memif_region_t *r; - clib_mem_vm_alloc_t alloc = { 0 }; clib_error_t *err; ASSERT (vec_len (mif->regions) == 0); @@ -323,29 +506,44 @@ memif_init_regions_and_queues (memif_if_t * mif) r->region_size += mif->run.buffer_size * (1 << mif->run.log2_ring_size) * (mif->run.num_s2m_rings + mif->run.num_m2s_rings); - alloc.name = "memif region"; - alloc.size = r->region_size; - alloc.flags = CLIB_MEM_VM_F_SHARED; + if ((fd = clib_mem_vm_create_fd (CLIB_MEM_PAGE_SZ_DEFAULT, "%U region 0", + format_memif_device_name, + mif->dev_instance)) == -1) + { + err = clib_mem_get_last_error (); + goto error; + } - err = clib_mem_vm_ext_alloc (&alloc); - if (err) - goto error; + if ((ftruncate (fd, r->region_size)) == -1) + { + err = clib_error_return_unix (0, "ftruncate"); + goto error; + } - r->fd = alloc.fd; - r->shm = alloc.addr; + msf = pool_elt_at_index (memif_main.socket_files, mif->socket_file_index); + r->shm = clib_mem_vm_map_shared (0, r->region_size, fd, 0, "memif%lu/%lu:0", + msf->socket_id, mif->id); + + if (r->shm == CLIB_MEM_VM_MAP_FAILED) + { + err = clib_error_return_unix (0, "memif shared region map failed"); + goto error; + } + + r->fd = fd; if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) { vlib_buffer_pool_t *bp; /* *INDENT-OFF* */ - vec_foreach (bp, buffer_main.buffer_pools) + vec_foreach (bp, vm->buffer_main->buffer_pools) { - vlib_physmem_region_t *pr; - pr = vlib_physmem_get_region (vm, bp->physmem_region); + vlib_physmem_map_t *pm; + pm = vlib_physmem_get_map (vm, bp->physmem_map_index); vec_add2_aligned (mif->regions, r, 1, CLIB_CACHE_LINE_BYTES); - r->fd = pr->fd; - r->region_size = pr->size; - r->shm = pr->mem; + r->fd = pm->fd; + r->region_size = pm->n_pages << pm->log2_page_size; + r->shm = pm->base; r->is_external = 1; } /* *INDENT-ON* */ @@ -402,6 +600,7 @@ memif_init_regions_and_queues (memif_if_t * mif) err = clib_error_return_unix (0, "eventfd[tx queue %u]", i); goto error; } + mq->int_clib_file_index = ~0; mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i); mq->log2_ring_size = mif->cfg.log2_ring_size; @@ -483,14 +682,16 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) case MEMIF_PROCESS_EVENT_STOP: enabled = 0; continue; + case MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN: + break; default: ASSERT (0); } last_run_duration = start_time = vlib_time_now (vm); /* *INDENT-OFF* */ - pool_foreach (mif, mm->interfaces, - ({ + pool_foreach (mif, mm->interfaces) + { memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index); /* Allow no more than 10us without a pause */ now = vlib_time_now (vm); @@ -513,9 +714,10 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { clib_memset (sock, 0, sizeof(clib_socket_t)); sock->config = (char *) msf->filename; - sock->flags = CLIB_SOCKET_F_IS_CLIENT| CLIB_SOCKET_F_SEQPACKET; + sock->is_seqpacket = 1; + sock->is_blocking = 1; - if ((err = clib_socket_init (sock))) + if ((err = clib_socket_init (sock))) { clib_error_free (err); } @@ -539,163 +741,162 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) sock = clib_mem_alloc (sizeof(clib_socket_t)); } } - })); + } /* *INDENT-ON* */ last_run_duration = vlib_time_now (vm) - last_run_duration; } return 0; } -/* *INDENT-OFF* */ VLIB_REGISTER_NODE (memif_process_node,static) = { .function = memif_process, .type = VLIB_NODE_TYPE_PROCESS, .name = "memif-process", }; -/* *INDENT-ON* */ -static int -memif_add_socket_file (u32 sock_id, u8 * socket_filename) +/* + * Returns an unused socket id, and ~0 if it can't find one. + */ +u32 +memif_get_unused_socket_id () { memif_main_t *mm = &memif_main; uword *p; - memif_socket_file_t *msf; + int i, j; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (p) + static u32 seed = 0; + /* limit to 1M tries */ + for (j = 0; j < 1 << 10; j++) { - msf = pool_elt_at_index (mm->socket_files, *p); - if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0) + seed = random_u32 (&seed); + for (i = 0; i < 1 << 10; i++) { - /* Silently accept identical "add". */ - return 0; + /* look around randomly generated id */ + seed += (2 * (i % 2) - 1) * i; + if (seed == (u32) ~0) + continue; + p = hash_get (mm->socket_file_index_by_sock_id, seed); + if (!p) + return seed; } - - /* But don't allow a direct add of a different filename. */ - return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; } - pool_get (mm->socket_files, msf); - clib_memset (msf, 0, sizeof (memif_socket_file_t)); - - msf->filename = socket_filename; - msf->socket_id = sock_id; - - hash_set (mm->socket_file_index_by_sock_id, sock_id, - msf - mm->socket_files); - - return 0; + return ~0; } -static int -memif_delete_socket_file (u32 sock_id) +clib_error_t * +memif_socket_filename_add_del (u8 is_add, u32 sock_id, char *sock_filename) { memif_main_t *mm = &memif_main; uword *p; memif_socket_file_t *msf; + clib_error_t *err = 0; + char *dir = 0, *tmp; + u32 idx = 0; + u8 *name = 0; - p = hash_get (mm->socket_file_index_by_sock_id, sock_id); - if (!p) - { - /* Don't delete non-existent entries. */ - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* allow adding socket id 0 */ + if (sock_id == 0 && is_add == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, "cannot delete socket id 0"); - msf = pool_elt_at_index (mm->socket_files, *p); - if (msf->ref_cnt > 0) - { - return VNET_API_ERROR_UNEXPECTED_INTF_STATE; - } + if (sock_id == ~0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socked id is not specified"); - vec_free (msf->filename); - pool_put (mm->socket_files, msf); + if (is_add == 0) + { + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (!p) + /* Don't delete non-existent entries. */ + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket file with id %u does not exist", sock_id); - hash_unset (mm->socket_file_index_by_sock_id, sock_id); + msf = pool_elt_at_index (mm->socket_files, *p); + if (msf->ref_cnt > 0) + return vnet_error (VNET_ERR_UNEXPECTED_INTF_STATE, + "socket file '%s' is in use", msf->filename); - return 0; -} + vec_free (msf->filename); + pool_put (mm->socket_files, msf); -int -memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename) -{ - char *dir = 0, *tmp; - u32 idx = 0; + hash_unset (mm->socket_file_index_by_sock_id, sock_id); - /* allow adding socket id 0 */ - if ((sock_id == 0 && is_add == 0) || sock_id == ~0) - { - return VNET_API_ERROR_INVALID_ARGUMENT; + return 0; } - if (is_add == 0) + if (sock_filename == 0 || sock_filename[0] == 0) + return vnet_error (VNET_ERR_INVALID_ARGUMENT, + "socket filename not specified"); + + if (clib_socket_prefix_is_valid (sock_filename)) { - return memif_delete_socket_file (sock_id); + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename == 0 || sock_filename[0] == 0) + else if (sock_filename[0] == '/') { - return VNET_API_ERROR_INVALID_ARGUMENT; + name = format (0, "%s%c", sock_filename, 0); } - - if (sock_filename[0] != '/') + else { - clib_error_t *error; - /* copy runtime dir path */ vec_add (dir, vlib_unix_get_runtime_dir (), strlen (vlib_unix_get_runtime_dir ())); vec_add1 (dir, '/'); /* if sock_filename contains dirs, add them to path */ - tmp = strrchr ((char *) sock_filename, '/'); + tmp = strrchr (sock_filename, '/'); if (tmp) { - idx = tmp - (char *) sock_filename; + idx = tmp - sock_filename; vec_add (dir, sock_filename, idx); } vec_add1 (dir, '\0'); /* create socket dir */ - error = vlib_unix_recursive_mkdir (dir); - if (error) + if ((err = vlib_unix_recursive_mkdir (dir))) { - clib_error_free (error); - return VNET_API_ERROR_SYSCALL_ERROR_1; + clib_error_free (err); + err = vnet_error (VNET_ERR_SYSCALL_ERROR_1, + "unable to create socket dir"); + goto done; } - sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), - sock_filename, 0); + name = + format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), sock_filename, 0); } - else - { - sock_filename = vec_dup (sock_filename); - /* check if directory exists */ - tmp = strrchr ((char *) sock_filename, '/'); - if (tmp) + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (p) + { + msf = pool_elt_at_index (mm->socket_files, *p); + if (strcmp ((char *) msf->filename, (char *) name) == 0) { - idx = tmp - (char *) sock_filename; - vec_add (dir, sock_filename, idx); - vec_add1 (dir, '\0'); + /* Silently accept identical "add". */ + goto done; } - /* check dir existance and access rights for effective user/group IDs */ - if ((dir == NULL) - || - (faccessat ( /* ignored */ -1, dir, F_OK | R_OK | W_OK, AT_EACCESS) - < 0)) - { - vec_free (dir); - return VNET_API_ERROR_INVALID_ARGUMENT; - } + /* But don't allow a direct add of a different filename. */ + err = vnet_error (VNET_ERR_ENTRY_ALREADY_EXISTS, "entry already exists"); + goto done; } - vec_free (dir); - return memif_add_socket_file (sock_id, sock_filename); + pool_get (mm->socket_files, msf); + clib_memset (msf, 0, sizeof (memif_socket_file_t)); + + msf->filename = name; + msf->socket_id = sock_id; + name = 0; + + hash_set (mm->socket_file_index_by_sock_id, sock_id, msf - mm->socket_files); + +done: + vec_free (name); + vec_free (dir); + return err; } -int -memif_delete_if (vlib_main_t * vm, memif_if_t * mif) +clib_error_t * +memif_delete_if (vlib_main_t *vm, memif_if_t *mif) { vnet_main_t *vnm = vnet_get_main (); memif_main_t *mm = &memif_main; @@ -726,7 +927,6 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) } /* free interface data structures */ - clib_spinlock_free (&mif->lockp); mhash_unset (&msf->dev_instance_by_id, &mif->id, 0); /* remove socket file */ @@ -756,6 +956,7 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) } } + vec_free (mif->local_disc_string); clib_memset (mif, 0, sizeof (*mif)); pool_put (mm->interfaces, mif); @@ -767,32 +968,40 @@ memif_delete_if (vlib_main_t * vm, memif_if_t * mif) } /* *INDENT-OFF* */ -VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = -{ +VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = { .name = "memif-ip", .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P, + .tx_hash_fn_type = VNET_HASH_FN_TYPE_IP, }; /* *INDENT-ON* */ -int -memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) +static void +memif_prepare_dma_args (vlib_dma_config_t *args) +{ + args->max_batches = 256; + args->max_transfer_size = VLIB_BUFFER_DEFAULT_DATA_SIZE; + args->barrier_before_last = 1; + args->sw_fallback = 1; + args->callback_fn = NULL; +} + +clib_error_t * +memif_create_if (vlib_main_t *vm, memif_create_if_args_t *args) { memif_main_t *mm = &memif_main; vlib_thread_main_t *tm = vlib_get_thread_main (); vnet_main_t *vnm = vnet_get_main (); + vnet_eth_interface_registration_t eir = {}; memif_if_t *mif = 0; vnet_sw_interface_t *sw; - clib_error_t *error = 0; - int ret = 0; uword *p; - vnet_hw_interface_t *hw; memif_socket_file_t *msf = 0; - int rv = 0; + clib_error_t *err = 0; p = hash_get (mm->socket_file_index_by_sock_id, args->socket_id); if (p == 0) { - rv = VNET_API_ERROR_INVALID_ARGUMENT; + err = vnet_error (VNET_ERR_INVALID_ARGUMENT, "unknown socket id"); goto done; } @@ -803,14 +1012,17 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) { if ((!msf->is_listener != !args->is_master)) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = + vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "socket file cannot be used by both master and slave"); goto done; } p = mhash_get (&msf->dev_instance_by_id, &args->id); if (p) { - rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + err = vnet_error (VNET_ERR_SUBIF_ALREADY_EXISTS, + "interface already exists"); goto done; } } @@ -818,25 +1030,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* Create new socket file */ if (msf->ref_cnt == 0) { - struct stat file_stat; - - /* If we are creating listener make sure file doesn't exist or if it - * exists thn delete it if it is old socket file */ - if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0)) - { - if (S_ISSOCK (file_stat.st_mode)) - { - unlink ((char *) msf->filename); - } - else - { - error = clib_error_return (0, "File exists for %s", - msf->filename); - rv = VNET_API_ERROR_VALUE_EXIST; - goto done; - } - } - mhash_init (&msf->dev_instance_by_id, sizeof (uword), sizeof (memif_interface_id_t)); msf->dev_instance_by_fd = hash_create (0, sizeof (uword)); @@ -848,25 +1041,20 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) if (mm->per_thread_data == 0) { int i; - vlib_buffer_free_list_t *fl; vec_validate_aligned (mm->per_thread_data, tm->n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); - fl = - vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); for (i = 0; i < tm->n_vlib_mains; i++) { memif_per_thread_data_t *ptd = vec_elt_at_index (mm->per_thread_data, i); vlib_buffer_t *bt = &ptd->buffer_template; - vlib_buffer_init_for_free_list (bt, fl); + clib_memset (bt, 0, sizeof (vlib_buffer_t)); bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; bt->total_length_not_including_first_buffer = 0; vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0; - /* initially prealloc copy_ops so we can use - _vec_len instead of vec_elen */ vec_validate_aligned (ptd->copy_ops, 0, CLIB_CACHE_LINE_BYTES); vec_reset_length (ptd->copy_ops); vec_validate_aligned (ptd->buffers, 0, CLIB_CACHE_LINE_BYTES); @@ -884,8 +1072,19 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) if (args->secret) mif->secret = vec_dup (args->secret); - if (tm->n_vlib_mains > 1) - clib_spinlock_init (&mif->lockp); + /* register dma config if enabled */ + if (args->use_dma) + { + vlib_dma_config_t dma_args; + bzero (&dma_args, sizeof (dma_args)); + memif_prepare_dma_args (&dma_args); + + dma_args.max_transfers = 1 << args->log2_ring_size; + dma_args.callback_fn = memif_dma_completion_cb; + mif->dma_input_config = vlib_dma_config_add (vm, &dma_args); + dma_args.callback_fn = memif_tx_dma_completion_cb; + mif->dma_tx_config = vlib_dma_config_add (vm, &dma_args); + } if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET) { @@ -901,10 +1100,13 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) args->hw_addr[0] = 2; args->hw_addr[1] = 0xfe; } - error = ethernet_register_interface (vnm, memif_device_class.index, - mif->dev_instance, args->hw_addr, - &mif->hw_if_index, - memif_eth_flag_change); + + eir.dev_class_index = memif_device_class.index; + eir.dev_instance = mif->dev_instance; + eir.address = args->hw_addr; + eir.cb.flag_change = memif_eth_flag_change; + eir.cb.set_max_frame_size = memif_eth_set_max_frame_size; + mif->hw_if_index = vnet_eth_register_interface (vnm, &eir); } else if (mif->mode == MEMIF_INTERFACE_MODE_IP) { @@ -915,11 +1117,9 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->dev_instance); } else - error = clib_error_return (0, "unsupported interface mode"); - - if (error) { - ret = VNET_API_ERROR_SYSCALL_ERROR_2; + err = + vnet_error (VNET_ERR_SYSCALL_ERROR_2, "unsupported interface mode"); goto error; } @@ -938,7 +1138,6 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) /* If this is new one, start listening */ if (msf->is_listener && msf->ref_cnt == 0) { - struct stat file_stat; clib_socket_t *s = clib_mem_alloc (sizeof (clib_socket_t)); ASSERT (msf->sock == 0); @@ -946,19 +1145,15 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) clib_memset (s, 0, sizeof (clib_socket_t)); s->config = (char *) msf->filename; - s->flags = CLIB_SOCKET_F_IS_SERVER | - CLIB_SOCKET_F_ALLOW_GROUP_WRITE | - CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED; - - if ((error = clib_socket_init (s))) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_4; - goto error; - } + s->local_only = 1; + s->is_server = 1; + s->allow_group_write = 1; + s->is_seqpacket = 1; + s->passcred = 1; - if (stat ((char *) msf->filename, &file_stat) == -1) + if ((err = clib_socket_init (s))) { - ret = VNET_API_ERROR_SYSCALL_ERROR_8; + err->code = VNET_ERR_SYSCALL_ERROR_4; goto error; } @@ -979,11 +1174,11 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) mif->flags |= MEMIF_IF_FLAG_ZERO_COPY; } - hw = vnet_get_hw_interface (vnm, mif->hw_if_index); - hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE; - vnet_hw_interface_set_input_node (vnm, mif->hw_if_index, - memif_input_node.index); + if (args->use_dma) + mif->flags |= MEMIF_IF_FLAG_USE_DMA; + vnet_hw_if_set_caps (vnm, mif->hw_if_index, VNET_HW_IF_CAP_INT_MODE); + vnet_hw_if_set_input_node (vnm, mif->hw_if_index, memif_input_node.index); mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0); if (pool_elts (mm->interfaces) == 1) @@ -994,24 +1189,38 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) goto done; error: - if (mif->hw_if_index != ~0) - { - if (mif->mode == MEMIF_INTERFACE_MODE_IP) - vnet_delete_hw_interface (vnm, mif->hw_if_index); - else - ethernet_delete_interface (vnm, mif->hw_if_index); - mif->hw_if_index = ~0; - } memif_delete_if (vm, mif); - if (error) + if (err) + memif_log_err (mif, "%U", format_clib_error, err); + return err; + +done: + return err; +} + +clib_error_t * +memif_interface_admin_up_down (vnet_main_t *vnm, u32 hw_if_index, u32 flags) +{ + memif_main_t *mm = &memif_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + memif_if_t *mif = pool_elt_at_index (mm->interfaces, hw->dev_instance); + static clib_error_t *error = 0; + + if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) { - memif_log_err (mif, "%U", format_clib_error, error); - clib_error_free (error); + if (mif->flags & MEMIF_IF_FLAG_CONNECTED) + { + vnet_hw_interface_set_flags (vnm, mif->hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); + } + mif->flags |= MEMIF_IF_FLAG_ADMIN_UP; } - return ret; + else + mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP; -done: - return rv; + vlib_process_signal_event (vnm->vlib_main, memif_process_node.index, + MEMIF_PROCESS_EVENT_ADMIN_UP_DOWN, 0); + return error; } static clib_error_t * @@ -1032,9 +1241,7 @@ memif_init (vlib_main_t * vm) * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the * default run-time directory. */ - memif_socket_filename_add_del (1, 0, (u8 *) MEMIF_DEFAULT_SOCKET_FILENAME); - - return 0; + return memif_socket_filename_add_del (1, 0, MEMIF_DEFAULT_SOCKET_FILENAME); } VLIB_INIT_FUNCTION (memif_init); @@ -1042,7 +1249,7 @@ VLIB_INIT_FUNCTION (memif_init); /* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "Packet Memory Interface (experimental)", + .description = "Packet Memory Interface (memif) -- Experimental", }; /* *INDENT-ON* */