X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fmemif%2Fmemif.c;h=e35a8dcb4a001abc586cd003214cd02455c582b0;hb=7a40133d3355463368085c14f9fc4b4fd22d549a;hp=7ba67c5b89817b53f30edb6fb2952f9a3492f88e;hpb=eaabe073515e7722ed546b36f99efc6feea305a1;p=vpp.git diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 7ba67c5b898..e35a8dcb4a0 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -1,6 +1,6 @@ /* *------------------------------------------------------------------ - * Copyright (c) 2016 Cisco and/or its affiliates. + * Copyright (c) 2017 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -15,6 +15,7 @@ *------------------------------------------------------------------ */ + #define _GNU_SOURCE #include #include @@ -26,29 +27,21 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include #include - -#define MEMIF_DEBUG 1 - -#if MEMIF_DEBUG == 1 -#define DEBUG_LOG(...) clib_warning(__VA_ARGS__) -#define DEBUG_UNIX_LOG(...) clib_unix_warning(__VA_ARGS__) -#else -#define DEBUG_LOG(...) -#endif +#include memif_main_t memif_main; -static clib_error_t *memif_conn_fd_read_ready (unix_file_t * uf); -static clib_error_t *memif_int_fd_read_ready (unix_file_t * uf); - static u32 memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) { @@ -57,529 +50,402 @@ memif_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) } static void -memif_remove_pending_conn (memif_pending_conn_t * pending_conn) -{ - memif_main_t *mm = &memif_main; - - unix_file_del (&unix_main, - unix_main.file_pool + pending_conn->connection.index); - pool_put (mm->pending_conns, pending_conn); -} - -static void -memif_connect (vlib_main_t * vm, memif_if_t * mif) +memif_queue_intfd_close (memif_queue_t * mq) { - vnet_main_t *vnm = vnet_get_main (); - int num_rings = mif->num_s2m_rings + mif->num_m2s_rings; - memif_ring_data_t *rd = NULL; - - vec_validate_aligned (mif->ring_data, num_rings - 1, CLIB_CACHE_LINE_BYTES); - vec_foreach (rd, mif->ring_data) - { - rd->last_head = 0; - } - - mif->flags &= ~MEMIF_IF_FLAG_CONNECTING; - mif->flags |= MEMIF_IF_FLAG_CONNECTED; - vnet_hw_interface_set_flags (vnm, mif->hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); -} - -static void -memif_disconnect (vlib_main_t * vm, memif_if_t * mif) -{ - vnet_main_t *vnm = vnet_get_main (); - - mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING); - if (mif->hw_if_index != ~0) - vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0); - - if (mif->interrupt_line.index != ~0) + if (mq->int_clib_file_index != ~0) { - unix_file_del (&unix_main, - unix_main.file_pool + mif->interrupt_line.index); - mif->interrupt_line.index = ~0; - mif->interrupt_line.fd = -1; /* closed in unix_file_del */ + memif_file_del_by_index (mq->int_clib_file_index); + mq->int_clib_file_index = ~0; + mq->int_fd = -1; } - if (mif->connection.index != ~0) + else if (mq->int_fd > -1) { - unix_file_del (&unix_main, unix_main.file_pool + mif->connection.index); - mif->connection.index = ~0; - mif->connection.fd = -1; /* closed in unix_file_del */ + close (mq->int_fd); + mq->int_fd = -1; } - - // TODO: properly munmap + close memif-owned shared memory segments - vec_free (mif->regions); } -static clib_error_t * -memif_process_connect_req (memif_pending_conn_t * pending_conn, - memif_msg_t * req, struct ucred *slave_cr, - int shm_fd, int int_fd) +void +memif_disconnect (memif_if_t * mif, clib_error_t * err) { memif_main_t *mm = &memif_main; - vlib_main_t *vm = vlib_get_main (); - int fd = pending_conn->connection.fd; - unix_file_t *uf = 0; - memif_if_t *mif = 0; - memif_msg_t resp = { 0 }; - unix_file_t template = { 0 }; - void *shm; - uword *p; - u8 retval = 0; - static clib_error_t *error = 0; - - if (shm_fd == -1) - { - DEBUG_LOG - ("Connection request is missing shared memory file descriptor"); - retval = 1; - goto response; - } + vnet_main_t *vnm = vnet_get_main (); + memif_region_t *mr; + memif_queue_t *mq; + int i; - if (int_fd == -1) - { - DEBUG_LOG - ("Connection request is missing interrupt line file descriptor"); - retval = 2; - goto response; - } + if (mif == 0) + return; - if (slave_cr == NULL) - { - DEBUG_LOG ("Connection request is missing slave credentials"); - retval = 3; - goto response; - } + memif_log_debug (mif, "disconnect %u (%v)", mif->dev_instance, + err ? err->what : 0); - p = mhash_get (&mm->if_index_by_key, &req->key); - if (!p) + if (err) { - DEBUG_LOG - ("Connection request with unmatched key (0x%" PRIx64 ")", req->key); - retval = 4; - goto response; + clib_error_t *e = 0; + mif->local_disc_string = vec_dup (err->what); + if (mif->sock && clib_socket_is_connected (mif->sock)) + e = memif_msg_send_disconnect (mif, err); + clib_error_free (e); } - mif = vec_elt_at_index (mm->interfaces, *p); - if (mif->listener_index != pending_conn->listener_index) - { - DEBUG_LOG - ("Connection request with non-matching listener (%d vs. %d)", - pending_conn->listener_index, mif->listener_index); - retval = 5; - goto response; - } + /* set interface down */ + mif->flags &= ~(MEMIF_IF_FLAG_CONNECTED | MEMIF_IF_FLAG_CONNECTING); + if (mif->hw_if_index != ~0) + vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0); - if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE) + /* close connection socket */ + if (mif->sock && mif->sock->fd) { - DEBUG_LOG ("Memif slave does not accept connection requests"); - retval = 6; - goto response; + memif_socket_file_t *msf = vec_elt_at_index (mm->socket_files, + mif->socket_file_index); + hash_unset (msf->dev_instance_by_fd, mif->sock->fd); + memif_socket_close (&mif->sock); } - - if (mif->connection.fd != -1) + else if (mif->sock) { - DEBUG_LOG - ("Memif with key 0x%" PRIx64 " is already connected", mif->key); - retval = 7; - goto response; + clib_error_t *err; + err = clib_socket_close (mif->sock); + if (err) + { + memif_log_err (mif, "%U", format_clib_error, err); + clib_error_free (err); + } + clib_mem_free (mif->sock); } - if ((mif->flags & MEMIF_IF_FLAG_ADMIN_UP) == 0) + /* *INDENT-OFF* */ + vec_foreach_index (i, mif->rx_queues) { - /* just silently decline the request */ - retval = 8; - goto response; + mq = vec_elt_at_index (mif->rx_queues, i); + if (mq->ring) + { + int rv; + rv = vnet_hw_interface_unassign_rx_thread (vnm, mif->hw_if_index, i); + if (rv) + memif_log_warn (mif, + "Unable to unassign interface %d, queue %d: rc=%d", + mif->hw_if_index, i, rv); + mq->ring = 0; + } } - if (req->shared_mem_size < sizeof (memif_shm_t)) - { - DEBUG_LOG - ("Unexpectedly small shared memory segment received from slave."); - retval = 9; - goto response; - } + /* free tx and rx queues */ + vec_foreach (mq, mif->rx_queues) + memif_queue_intfd_close (mq); + vec_free (mif->rx_queues); - if ((shm = - mmap (NULL, req->shared_mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, - shm_fd, 0)) == MAP_FAILED) - { - DEBUG_UNIX_LOG - ("Failed to map shared memory segment received from slave memif"); - error = clib_error_return_unix (0, "mmap fd %d", shm_fd); - retval = 10; - goto response; - } + vec_foreach (mq, mif->tx_queues) + memif_queue_intfd_close (mq); + vec_free (mif->tx_queues); - if (((memif_shm_t *) shm)->cookie != 0xdeadbeef) + /* free memory regions */ + vec_foreach (mr, mif->regions) { - DEBUG_LOG - ("Possibly corrupted shared memory segment received from slave memif"); - munmap (shm, req->shared_mem_size); - retval = 11; - goto response; + int rv; + if (mr->is_external) + continue; + if ((rv = munmap (mr->shm, mr->region_size))) + memif_log_err (mif, "munmap failed, rv = %d", rv); + if (mr->fd > -1) + close (mr->fd); } - - mif->log2_ring_size = req->log2_ring_size; - mif->num_s2m_rings = req->num_s2m_rings; - mif->num_m2s_rings = req->num_m2s_rings; - mif->buffer_size = req->buffer_size; - mif->remote_pid = slave_cr->pid; - mif->remote_uid = slave_cr->uid; - vec_add1 (mif->regions, shm); - - /* register interrupt line */ - mif->interrupt_line.fd = int_fd; - template.read_function = memif_int_fd_read_ready; - template.file_descriptor = int_fd; - template.private_data = mif->if_index; - mif->interrupt_line.index = unix_file_add (&unix_main, &template); - - /* change context for future messages */ - uf = vec_elt_at_index (unix_main.file_pool, pending_conn->connection.index); - uf->private_data = mif->if_index << 1; - mif->connection = pending_conn->connection; - pool_put (mm->pending_conns, pending_conn); - - memif_connect (vm, mif); - -response: - resp.version = MEMIF_VERSION; - resp.type = MEMIF_MSG_TYPE_CONNECT_RESP; - resp.retval = retval; - send (fd, &resp, sizeof (resp), 0); - return error; + /* *INDENT-ON* */ + vec_free (mif->regions); + vec_free (mif->remote_name); + vec_free (mif->remote_if_name); + clib_fifo_free (mif->msg_queue); } static clib_error_t * -memif_process_connect_resp (memif_if_t * mif, memif_msg_t * resp) +memif_int_fd_read_ready (clib_file_t * uf) { - vlib_main_t *vm = vlib_get_main (); - - if ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) == 0) - { - DEBUG_LOG ("Memif master does not accept connection responses"); - return 0; - } + memif_main_t *mm = &memif_main; + vnet_main_t *vnm = vnet_get_main (); + u16 qid = uf->private_data & 0xFFFF; + memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 16); + memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, qid); + u64 b; + ssize_t size; - if ((mif->flags & MEMIF_IF_FLAG_CONNECTING) == 0) + size = read (uf->file_descriptor, &b, sizeof (b)); + if (size < 0) { - DEBUG_LOG ("Unexpected connection response"); + memif_log_debug (mif, "Failed to read form socket"); return 0; } - if (resp->retval == 0) - memif_connect (vm, mif); - else - memif_disconnect (vm, mif); + vnet_device_input_set_interrupt_pending (vnm, mif->hw_if_index, qid); + mq->int_count++; return 0; } -static clib_error_t * -memif_conn_fd_read_ready (unix_file_t * uf) + +clib_error_t * +memif_connect (memif_if_t * mif) { - memif_main_t *mm = &memif_main; - vlib_main_t *vm = vlib_get_main (); - memif_if_t *mif = 0; - memif_pending_conn_t *pending_conn = 0; - int fd_array[2] = { -1, -1 }; - char ctl[CMSG_SPACE (sizeof (fd_array)) + - CMSG_SPACE (sizeof (struct ucred))] = { 0 }; - struct msghdr mh = { 0 }; - struct iovec iov[1]; - struct ucred *cr = 0; - memif_msg_t msg = { 0 }; - struct cmsghdr *cmsg; - ssize_t size; - static clib_error_t *error = 0; - - iov[0].iov_base = (void *) &msg; - iov[0].iov_len = sizeof (memif_msg_t); - mh.msg_iov = iov; - mh.msg_iovlen = 1; - mh.msg_control = ctl; - mh.msg_controllen = sizeof (ctl); - - /* grab the appropriate context */ - if (uf->private_data & 1) - pending_conn = vec_elt_at_index (mm->pending_conns, - uf->private_data >> 1); - else - mif = vec_elt_at_index (mm->interfaces, uf->private_data >> 1); + vnet_main_t *vnm = vnet_get_main (); + clib_file_t template = { 0 }; + memif_region_t *mr; + int i; + clib_error_t *err = NULL; - /* receive the incoming message */ - size = recvmsg (uf->file_descriptor, &mh, 0); - if (size != sizeof (memif_msg_t)) + memif_log_debug (mif, "connect %u", mif->dev_instance); + + vec_free (mif->local_disc_string); + vec_free (mif->remote_disc_string); + + /* *INDENT-OFF* */ + vec_foreach (mr, mif->regions) { - if (size != 0) + if (mr->shm) + continue; + + if (mr->fd < 0) { - DEBUG_UNIX_LOG ("Malformed message received on fd %d", - uf->file_descriptor); - error = clib_error_return_unix (0, "recvmsg fd %d", - uf->file_descriptor); + err = clib_error_return (0, "no memory region fd"); + goto error; } - goto disconnect; - } - /* check version of the sender's memif plugin */ - if (msg.version != MEMIF_VERSION) - { - DEBUG_LOG ("Memif version mismatch"); - goto disconnect; + if ((mr->shm = mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE, + MAP_SHARED, mr->fd, 0)) == MAP_FAILED) + { + err = clib_error_return_unix (0, "mmap"); + goto error; + } } + /* *INDENT-ON* */ - /* process the message based on its type */ - switch (msg.type) + template.read_function = memif_int_fd_read_ready; + + /* *INDENT-OFF* */ + vec_foreach_index (i, mif->tx_queues) { - case MEMIF_MSG_TYPE_CONNECT_REQ: - if (pending_conn == 0) - { - DEBUG_LOG ("Received unexpected connection request"); - return 0; - } + memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i); - /* Read anciliary data */ - cmsg = CMSG_FIRSTHDR (&mh); - while (cmsg) + mq->ring = mif->regions[mq->region].shm + mq->offset; + if (mq->ring->cookie != MEMIF_COOKIE) { - if (cmsg->cmsg_level == SOL_SOCKET - && cmsg->cmsg_type == SCM_CREDENTIALS) - { - cr = (struct ucred *) CMSG_DATA (cmsg); - } - else if (cmsg->cmsg_level == SOL_SOCKET - && cmsg->cmsg_type == SCM_RIGHTS) - { - clib_memcpy (fd_array, CMSG_DATA (cmsg), sizeof (fd_array)); - } - cmsg = CMSG_NXTHDR (&mh, cmsg); + err = clib_error_return (0, "wrong cookie on tx ring %u", i); + goto error; } + } - return memif_process_connect_req (pending_conn, &msg, cr, - fd_array[0], fd_array[1]); + vec_foreach_index (i, mif->rx_queues) + { + memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i); + int rv; - case MEMIF_MSG_TYPE_CONNECT_RESP: - if (mif == 0) + mq->ring = mif->regions[mq->region].shm + mq->offset; + if (mq->ring->cookie != MEMIF_COOKIE) { - DEBUG_LOG ("Received unexpected connection response"); - return 0; + err = clib_error_return (0, "wrong cookie on tx ring %u", i); + goto error; } - return memif_process_connect_resp (mif, &msg); - case MEMIF_MSG_TYPE_DISCONNECT: - goto disconnect; + if (mq->int_fd > -1) + { + template.file_descriptor = mq->int_fd; + template.private_data = (mif->dev_instance << 16) | (i & 0xFFFF); + template.description = format (0, "%U rx %u int", + format_memif_device_name, + mif->dev_instance, i); + memif_file_add (&mq->int_clib_file_index, &template); + } + vnet_hw_interface_assign_rx_thread (vnm, mif->hw_if_index, i, ~0); + rv = vnet_hw_interface_set_rx_mode (vnm, mif->hw_if_index, i, + VNET_HW_INTERFACE_RX_MODE_DEFAULT); + if (rv) + memif_log_err + (mif, "Warning: unable to set rx mode for interface %d queue %d: " + "rc=%d", mif->hw_if_index, i, rv); + else + { + vnet_hw_interface_rx_mode rxmode; + vnet_hw_interface_get_rx_mode (vnm, mif->hw_if_index, i, &rxmode); - default: - DEBUG_LOG ("Received unknown message type"); - goto disconnect; + if (rxmode == VNET_HW_INTERFACE_RX_MODE_POLLING) + mq->ring->flags |= MEMIF_RING_FLAG_MASK_INT; + else + vnet_device_input_set_interrupt_pending (vnm, mif->hw_if_index, i); + } } + /* *INDENT-ON* */ - return 0; - -disconnect: - if (pending_conn) - memif_remove_pending_conn (pending_conn); - else - memif_disconnect (vm, mif); - return error; -} - -static clib_error_t * -memif_int_fd_read_ready (unix_file_t * uf) -{ - memif_main_t *mm = &memif_main; - vlib_main_t *vm = vlib_get_main (); - memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data); - u8 b; - ssize_t size; + mif->flags &= ~MEMIF_IF_FLAG_CONNECTING; + mif->flags |= MEMIF_IF_FLAG_CONNECTED; - size = read (uf->file_descriptor, &b, sizeof (b)); - if (0 == size) - { - /* interrupt line was disconnected */ - unix_file_del (&unix_main, - unix_main.file_pool + mif->interrupt_line.index); - mif->interrupt_line.index = ~0; - mif->interrupt_line.fd = -1; - } - vlib_node_set_interrupt_pending (vm, memif_input_node.index); + vnet_hw_interface_set_flags (vnm, mif->hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); return 0; + +error: + memif_log_err (mif, "%U", format_clib_error, err); + return err; } -static clib_error_t * -memif_conn_fd_accept_ready (unix_file_t * uf) +static_always_inline memif_ring_t * +memif_get_ring (memif_if_t * mif, memif_ring_type_t type, u16 ring_num) { - memif_main_t *mm = &memif_main; - memif_listener_t *listener = 0; - memif_pending_conn_t *pending_conn = 0; - int addr_len; - struct sockaddr_un client; - int conn_fd; - unix_file_t template = { 0 }; - - listener = pool_elt_at_index (mm->listeners, uf->private_data); - - addr_len = sizeof (client); - conn_fd = accept (uf->file_descriptor, - (struct sockaddr *) &client, (socklen_t *) & addr_len); - - if (conn_fd < 0) - return clib_error_return_unix (0, "accept fd %d", uf->file_descriptor); - - pool_get (mm->pending_conns, pending_conn); - pending_conn->index = pending_conn - mm->pending_conns; - pending_conn->listener_index = listener->index; - pending_conn->connection.fd = conn_fd; - - template.read_function = memif_conn_fd_read_ready; - template.file_descriptor = conn_fd; - template.private_data = (pending_conn->index << 1) | 1; - pending_conn->connection.index = unix_file_add (&unix_main, &template); - - return 0; + if (vec_len (mif->regions) == 0) + return NULL; + void *p = mif->regions[0].shm; + int ring_size = + sizeof (memif_ring_t) + + sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size); + p += (ring_num + type * mif->run.num_s2m_rings) * ring_size; + + return (memif_ring_t *) p; } -static void -memif_connect_master (vlib_main_t * vm, memif_if_t * mif) +clib_error_t * +memif_init_regions_and_queues (memif_if_t * mif) { - memif_msg_t msg; - struct msghdr mh = { 0 }; - struct iovec iov[1]; - struct cmsghdr *cmsg; - int mfd = -1; - int rv; - int fd_array[2] = { -1, -1 }; - char ctl[CMSG_SPACE (sizeof (fd_array))]; + vlib_main_t *vm = vlib_get_main (); memif_ring_t *ring = NULL; int i, j; - void *shm = 0; u64 buffer_offset; - unix_file_t template = { 0 }; - - msg.version = MEMIF_VERSION; - msg.type = MEMIF_MSG_TYPE_CONNECT_REQ; - msg.key = mif->key; - msg.log2_ring_size = mif->log2_ring_size; - msg.num_s2m_rings = mif->num_s2m_rings; - msg.num_m2s_rings = mif->num_m2s_rings; - msg.buffer_size = mif->buffer_size; - - buffer_offset = sizeof (memif_shm_t) + - (mif->num_s2m_rings + mif->num_m2s_rings) * + memif_region_t *r; + clib_mem_vm_alloc_t alloc = { 0 }; + clib_error_t *err; + + ASSERT (vec_len (mif->regions) == 0); + vec_add2_aligned (mif->regions, r, 1, CLIB_CACHE_LINE_BYTES); + + buffer_offset = (mif->run.num_s2m_rings + mif->run.num_m2s_rings) * (sizeof (memif_ring_t) + - sizeof (memif_desc_t) * (1 << mif->log2_ring_size)); + sizeof (memif_desc_t) * (1 << mif->run.log2_ring_size)); - msg.shared_mem_size = buffer_offset + - mif->buffer_size * (1 << mif->log2_ring_size) * (mif->num_s2m_rings + - mif->num_m2s_rings); + r->region_size = buffer_offset; - if ((mfd = memfd_create ("shared mem", MFD_ALLOW_SEALING)) == -1) - { - DEBUG_LOG ("Failed to create anonymous file"); - goto error; - } + if ((mif->flags & MEMIF_IF_FLAG_ZERO_COPY) == 0) + r->region_size += mif->run.buffer_size * (1 << mif->run.log2_ring_size) * + (mif->run.num_s2m_rings + mif->run.num_m2s_rings); - if ((fcntl (mfd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) - { - DEBUG_UNIX_LOG ("Failed to seal an anonymous file off from truncating"); - goto error; - } + alloc.name = "memif region"; + alloc.size = r->region_size; + alloc.flags = CLIB_MEM_VM_F_SHARED; - if ((ftruncate (mfd, msg.shared_mem_size)) == -1) - { - DEBUG_UNIX_LOG ("Failed to extend the size of an anonymous file"); - goto error; - } + err = clib_mem_vm_ext_alloc (&alloc); + if (err) + goto error; - if ((shm = mmap (NULL, msg.shared_mem_size, PROT_READ | PROT_WRITE, - MAP_SHARED, mfd, 0)) == MAP_FAILED) + r->fd = alloc.fd; + r->shm = alloc.addr; + + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) { - DEBUG_UNIX_LOG ("Failed to map anonymous file into memory"); - goto error; + vlib_buffer_pool_t *bp; + /* *INDENT-OFF* */ + vec_foreach (bp, buffer_main.buffer_pools) + { + vlib_physmem_region_t *pr; + pr = vlib_physmem_get_region (vm, bp->physmem_region); + vec_add2_aligned (mif->regions, r, 1, CLIB_CACHE_LINE_BYTES); + r->fd = pr->fd; + r->region_size = pr->size; + r->shm = pr->mem; + r->is_external = 1; + } + /* *INDENT-ON* */ } - vec_add1 (mif->regions, shm); - ((memif_shm_t *) mif->regions[0])->cookie = 0xdeadbeef; - - for (i = 0; i < mif->num_s2m_rings; i++) + for (i = 0; i < mif->run.num_s2m_rings; i++) { ring = memif_get_ring (mif, MEMIF_RING_S2M, i); ring->head = ring->tail = 0; - for (j = 0; j < (1 << mif->log2_ring_size); j++) + ring->cookie = MEMIF_COOKIE; + + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + continue; + + for (j = 0; j < (1 << mif->run.log2_ring_size); j++) { - u16 slot = i * (1 << mif->log2_ring_size) + j; + u16 slot = i * (1 << mif->run.log2_ring_size) + j; ring->desc[j].region = 0; - ring->desc[j].offset = buffer_offset + (slot * mif->buffer_size); - ring->desc[j].buffer_length = mif->buffer_size; + ring->desc[j].offset = + buffer_offset + (u32) (slot * mif->run.buffer_size); + ring->desc[j].length = mif->run.buffer_size; } } - for (i = 0; i < mif->num_m2s_rings; i++) + for (i = 0; i < mif->run.num_m2s_rings; i++) { ring = memif_get_ring (mif, MEMIF_RING_M2S, i); ring->head = ring->tail = 0; - for (j = 0; j < (1 << mif->log2_ring_size); j++) + ring->cookie = MEMIF_COOKIE; + + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + continue; + + for (j = 0; j < (1 << mif->run.log2_ring_size); j++) { u16 slot = - (i + mif->num_s2m_rings) * (1 << mif->log2_ring_size) + j; + (i + mif->run.num_s2m_rings) * (1 << mif->run.log2_ring_size) + j; ring->desc[j].region = 0; - ring->desc[j].offset = buffer_offset + (slot * mif->buffer_size); - ring->desc[j].buffer_length = mif->buffer_size; + ring->desc[j].offset = + buffer_offset + (u32) (slot * mif->run.buffer_size); + ring->desc[j].length = mif->run.buffer_size; } } - iov[0].iov_base = (void *) &msg; - iov[0].iov_len = sizeof (memif_msg_t); - mh.msg_iov = iov; - mh.msg_iovlen = 1; + ASSERT (mif->tx_queues == 0); + vec_validate_aligned (mif->tx_queues, mif->run.num_s2m_rings - 1, + CLIB_CACHE_LINE_BYTES); - /* create interrupt socket */ - if (socketpair (AF_UNIX, SOCK_STREAM, 0, fd_array) < 0) + /* *INDENT-OFF* */ + vec_foreach_index (i, mif->tx_queues) { - DEBUG_UNIX_LOG ("Failed to create a pair of connected sockets"); - goto error; + memif_queue_t *mq = vec_elt_at_index (mif->tx_queues, i); + if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0) + { + err = clib_error_return_unix (0, "eventfd[tx queue %u]", i); + goto error; + } + mq->int_clib_file_index = ~0; + mq->ring = memif_get_ring (mif, MEMIF_RING_S2M, i); + mq->log2_ring_size = mif->cfg.log2_ring_size; + mq->region = 0; + mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm; + mq->last_head = 0; + mq->type = MEMIF_RING_S2M; + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size, + CLIB_CACHE_LINE_BYTES); } + /* *INDENT-ON* */ - mif->interrupt_line.fd = fd_array[0]; - template.read_function = memif_int_fd_read_ready; - template.file_descriptor = mif->interrupt_line.fd; - template.private_data = mif->if_index; - mif->interrupt_line.index = unix_file_add (&unix_main, &template); - - memset (&ctl, 0, sizeof (ctl)); - mh.msg_control = ctl; - mh.msg_controllen = sizeof (ctl); - cmsg = CMSG_FIRSTHDR (&mh); - cmsg->cmsg_len = CMSG_LEN (sizeof (fd_array)); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - fd_array[0] = mfd; - clib_memcpy (CMSG_DATA (cmsg), fd_array, sizeof (fd_array)); - - mif->flags |= MEMIF_IF_FLAG_CONNECTING; - rv = sendmsg (mif->connection.fd, &mh, 0); - if (rv < 0) + ASSERT (mif->rx_queues == 0); + vec_validate_aligned (mif->rx_queues, mif->run.num_m2s_rings - 1, + CLIB_CACHE_LINE_BYTES); + + /* *INDENT-OFF* */ + vec_foreach_index (i, mif->rx_queues) { - DEBUG_UNIX_LOG ("Failed to send memif connection request"); - goto error; + memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i); + if ((mq->int_fd = eventfd (0, EFD_NONBLOCK)) < 0) + { + err = clib_error_return_unix (0, "eventfd[rx queue %u]", i); + goto error; + } + mq->int_clib_file_index = ~0; + mq->ring = memif_get_ring (mif, MEMIF_RING_M2S, i); + mq->log2_ring_size = mif->cfg.log2_ring_size; + mq->region = 0; + mq->offset = (void *) mq->ring - (void *) mif->regions[mq->region].shm; + mq->last_head = 0; + mq->type = MEMIF_RING_M2S; + if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY) + vec_validate_aligned (mq->buffers, 1 << mq->log2_ring_size, + CLIB_CACHE_LINE_BYTES); } + /* *INDENT-ON* */ - /* No need to keep the descriptor open, - * mmap creates an extra reference to the underlying file */ - close (mfd); - mfd = -1; - /* This FD is given to peer, so we can close it */ - close (fd_array[1]); - fd_array[1] = -1; - return; + return 0; error: - if (mfd > -1) - close (mfd); - if (fd_array[1] > -1) - close (fd_array[1]); - memif_disconnect (vm, mif); + memif_log_err (mif, "%U", format_clib_error, err); + return err; } static uword @@ -587,22 +453,20 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { memif_main_t *mm = &memif_main; memif_if_t *mif; - struct sockaddr_un sun; - int sockfd; + clib_socket_t *sock; uword *event_data = 0, event_type; - unix_file_t template = { 0 }; u8 enabled = 0; f64 start_time, last_run_duration = 0, now; + clib_error_t *err; - sockfd = socket (AF_UNIX, SOCK_STREAM, 0); - sun.sun_family = AF_UNIX; - template.read_function = memif_conn_fd_read_ready; + sock = clib_mem_alloc (sizeof (clib_socket_t)); + memset (sock, 0, sizeof (clib_socket_t)); while (1) { if (enabled) - vlib_process_wait_for_event_or_clock (vm, - (f64) 3 - last_run_duration); + vlib_process_wait_for_event_or_clock (vm, (f64) 3 - + last_run_duration); else vlib_process_wait_for_event (vm); @@ -627,6 +491,7 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) /* *INDENT-OFF* */ pool_foreach (mif, mm->interfaces, ({ + memif_socket_file_t * msf = vec_elt_at_index (mm->socket_files, mif->socket_file_index); /* Allow no more than 10us without a pause */ now = vlib_time_now (vm); if (now > start_time + 10e-6) @@ -646,26 +511,32 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE) { - strncpy (sun.sun_path, (char *) mif->socket_filename, - sizeof (sun.sun_path) - 1); - - if (connect - (sockfd, (struct sockaddr *) &sun, - sizeof (struct sockaddr_un)) == 0) + memset (sock, 0, sizeof(clib_socket_t)); + sock->config = (char *) msf->filename; + sock->flags = CLIB_SOCKET_F_IS_CLIENT| CLIB_SOCKET_F_SEQPACKET; + + if ((err = clib_socket_init (sock))) + { + clib_error_free (err); + } + else { - mif->connection.fd = sockfd; - template.file_descriptor = sockfd; - template.private_data = mif->if_index << 1; - mif->connection.index = unix_file_add (&unix_main, &template); - memif_connect_master (vm, mif); - - /* grab another fd */ - sockfd = socket (AF_UNIX, SOCK_STREAM, 0); - if (sockfd < 0) - { - DEBUG_UNIX_LOG ("socket AF_UNIX"); - return 0; - } + clib_file_t t = { 0 }; + + t.read_function = memif_slave_conn_fd_read_ready; + t.write_function = memif_slave_conn_fd_write_ready; + t.error_function = memif_slave_conn_fd_error; + t.file_descriptor = sock->fd; + t.private_data = mif->dev_instance; + memif_file_add (&sock->private_data, &t); + t.description = format (0, "%U ctl", + format_memif_device_name, + mif->dev_instance); + hash_set (msf->dev_instance_by_fd, sock->fd, mif->dev_instance); + + mif->flags |= MEMIF_IF_FLAG_CONNECTING; + mif->sock = sock; + sock = clib_mem_alloc (sizeof(clib_socket_t)); } } })); @@ -683,80 +554,224 @@ VLIB_REGISTER_NODE (memif_process_node,static) = { }; /* *INDENT-ON* */ -static void -memif_close_if (memif_main_t * mm, memif_if_t * mif) +static int +memif_add_socket_file (u32 sock_id, u8 * socket_filename) { - vlib_main_t *vm = vlib_get_main (); - memif_listener_t *listener = 0; - memif_pending_conn_t *pending_conn = 0; - - memif_disconnect (vm, mif); + memif_main_t *mm = &memif_main; + uword *p; + memif_socket_file_t *msf; - if (mif->listener_index != (uword) ~ 0) + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (p) { - listener = pool_elt_at_index (mm->listeners, mif->listener_index); - if (--listener->usage_counter == 0) + msf = pool_elt_at_index (mm->socket_files, *p); + if (strcmp ((char *) msf->filename, (char *) socket_filename) == 0) { - /* not used anymore -> remove the socket and pending connections */ - - /* *INDENT-OFF* */ - pool_foreach (pending_conn, mm->pending_conns, - ({ - if (pending_conn->listener_index == mif->listener_index) - { - memif_remove_pending_conn (pending_conn); - } - })); - /* *INDENT-ON* */ - - unix_file_del (&unix_main, - unix_main.file_pool + listener->socket.index); - pool_put (mm->listeners, listener); - unlink ((char *) mif->socket_filename); + /* Silently accept identical "add". */ + return 0; } + + /* But don't allow a direct add of a different filename. */ + return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; } - if (mif->lockp != 0) + pool_get (mm->socket_files, msf); + memset (msf, 0, sizeof (memif_socket_file_t)); + + msf->filename = socket_filename; + msf->socket_id = sock_id; + + hash_set (mm->socket_file_index_by_sock_id, sock_id, + msf - mm->socket_files); + + return 0; +} + +static int +memif_delete_socket_file (u32 sock_id) +{ + memif_main_t *mm = &memif_main; + uword *p; + memif_socket_file_t *msf; + + p = hash_get (mm->socket_file_index_by_sock_id, sock_id); + if (!p) { - clib_mem_free ((void *) mif->lockp); - mif->lockp = 0; + /* Don't delete non-existent entries. */ + return VNET_API_ERROR_INVALID_ARGUMENT; } - mhash_unset (&mm->if_index_by_key, &mif->key, &mif->if_index); - vec_free (mif->socket_filename); - vec_free (mif->ring_data); + msf = pool_elt_at_index (mm->socket_files, *p); + if (msf->ref_cnt > 0) + { + return VNET_API_ERROR_UNEXPECTED_INTF_STATE; + } - memset (mif, 0, sizeof (*mif)); - pool_put (mm->interfaces, mif); + vec_free (msf->filename); + pool_put (mm->socket_files, msf); + + hash_unset (mm->socket_file_index_by_sock_id, sock_id); + + return 0; } int -memif_worker_thread_enable () +memif_socket_filename_add_del (u8 is_add, u32 sock_id, u8 * sock_filename) { - /* if worker threads are enabled, switch to polling mode */ - foreach_vlib_main (( - { - vlib_node_set_state (this_vlib_main, - memif_input_node.index, - VLIB_NODE_STATE_POLLING); - })); + struct stat file_stat; + char *dir = 0, *tmp; + u32 idx = 0; - return 0; + /* allow adding socket id 0 */ + if ((sock_id == 0 && is_add == 0) || sock_id == ~0) + { + return VNET_API_ERROR_INVALID_ARGUMENT; + } + + if (is_add == 0) + { + return memif_delete_socket_file (sock_id); + } + + if (sock_filename == 0 || sock_filename[0] == 0) + { + return VNET_API_ERROR_INVALID_ARGUMENT; + } + + if (sock_filename[0] != '/') + { + clib_error_t *error; + + /* copy runtime dir path */ + vec_add (dir, vlib_unix_get_runtime_dir (), + strlen (vlib_unix_get_runtime_dir ())); + vec_add1 (dir, '/'); + + /* if sock_filename contains dirs, add them to path */ + tmp = strrchr ((char *) sock_filename, '/'); + if (tmp) + { + idx = tmp - (char *) sock_filename; + vec_add (dir, sock_filename, idx); + } + + vec_add1 (dir, '\0'); + /* create socket dir */ + error = vlib_unix_recursive_mkdir (dir); + if (error) + { + clib_error_free (error); + return VNET_API_ERROR_SYSCALL_ERROR_1; + } + + sock_filename = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (), + sock_filename, 0); + } + else + { + sock_filename = vec_dup (sock_filename); + + /* check if directory exists */ + tmp = strrchr ((char *) sock_filename, '/'); + if (tmp) + { + idx = tmp - (char *) sock_filename; + vec_add (dir, sock_filename, idx); + vec_add1 (dir, '\0'); + } + + if (((dir == 0) || (stat (dir, &file_stat) == -1) + || (!S_ISDIR (file_stat.st_mode))) && (idx != 0)) + { + vec_free (dir); + return VNET_API_ERROR_INVALID_ARGUMENT; + } + } + vec_free (dir); + + return memif_add_socket_file (sock_id, sock_filename); } int -memif_worker_thread_disable () +memif_delete_if (vlib_main_t * vm, memif_if_t * mif) { - foreach_vlib_main (( - { - vlib_node_set_state (this_vlib_main, - memif_input_node.index, - VLIB_NODE_STATE_INTERRUPT); - })); + vnet_main_t *vnm = vnet_get_main (); + memif_main_t *mm = &memif_main; + memif_socket_file_t *msf = + vec_elt_at_index (mm->socket_files, mif->socket_file_index); + clib_error_t *err; + + mif->flags |= MEMIF_IF_FLAG_DELETING; + vec_free (mif->local_disc_string); + vec_free (mif->remote_disc_string); + + /* bring down the interface */ + vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0); + vnet_sw_interface_set_flags (vnm, mif->sw_if_index, 0); + + err = clib_error_return (0, "interface deleted"); + memif_disconnect (mif, err); + clib_error_free (err); + + if (mif->hw_if_index != ~0) + { + /* remove the interface */ + if (mif->mode == MEMIF_INTERFACE_MODE_IP) + vnet_delete_hw_interface (vnm, mif->hw_if_index); + else + ethernet_delete_interface (vnm, mif->hw_if_index); + mif->hw_if_index = ~0; + } + + /* free interface data structures */ + clib_spinlock_free (&mif->lockp); + mhash_unset (&msf->dev_instance_by_id, &mif->id, 0); + + /* remove socket file */ + if (--(msf->ref_cnt) == 0) + { + if (msf->is_listener) + { + int i; + /* *INDENT-OFF* */ + vec_foreach_index (i, msf->pending_clients) + memif_socket_close (msf->pending_clients + i); + /* *INDENT-ON* */ + memif_socket_close (&msf->sock); + vec_free (msf->pending_clients); + } + mhash_free (&msf->dev_instance_by_id); + hash_free (msf->dev_instance_by_fd); + if (msf->sock) + { + err = clib_socket_close (msf->sock); + if (err) + { + memif_log_err (mif, "%U", format_clib_error, err); + clib_error_free (err); + } + clib_mem_free (msf->sock); + } + } + + memset (mif, 0, sizeof (*mif)); + pool_put (mm->interfaces, mif); + + if (pool_elts (mm->interfaces) == 0) + vlib_process_signal_event (vm, memif_process_node.index, + MEMIF_PROCESS_EVENT_STOP, 0); return 0; } +/* *INDENT-OFF* */ +VNET_HW_INTERFACE_CLASS (memif_ip_hw_if_class, static) = +{ + .name = "memif-ip", + .flags = VNET_HW_INTERFACE_CLASS_FLAG_P2P, +}; +/* *INDENT-ON* */ + int memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) { @@ -768,261 +783,254 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args) clib_error_t *error = 0; int ret = 0; uword *p; + vnet_hw_interface_t *hw; + memif_socket_file_t *msf = 0; + int rv = 0; - p = mhash_get (&mm->if_index_by_key, &args->key); - if (p) - return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + p = hash_get (mm->socket_file_index_by_sock_id, args->socket_id); + if (p == 0) + { + rv = VNET_API_ERROR_INVALID_ARGUMENT; + goto done; + } + + msf = vec_elt_at_index (mm->socket_files, p[0]); + + /* existing socket file can be either master or slave but cannot be both */ + if (msf->ref_cnt > 0) + { + if ((!msf->is_listener != !args->is_master)) + { + rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + goto done; + } + + p = mhash_get (&msf->dev_instance_by_id, &args->id); + if (p) + { + rv = VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + goto done; + } + } + + /* Create new socket file */ + if (msf->ref_cnt == 0) + { + struct stat file_stat; + + /* If we are creating listener make sure file doesn't exist or if it + * exists thn delete it if it is old socket file */ + if (args->is_master && (stat ((char *) msf->filename, &file_stat) == 0)) + { + if (S_ISSOCK (file_stat.st_mode)) + { + unlink ((char *) msf->filename); + } + else + { + error = clib_error_return (0, "File exists for %s", + msf->filename); + rv = VNET_API_ERROR_VALUE_EXIST; + goto done; + } + } + + mhash_init (&msf->dev_instance_by_id, sizeof (uword), + sizeof (memif_interface_id_t)); + msf->dev_instance_by_fd = hash_create (0, sizeof (uword)); + msf->is_listener = (args->is_master != 0); + + memif_log_debug (0, "initializing socket file %s", msf->filename); + } + + if (mm->per_thread_data == 0) + { + int i; + vlib_buffer_free_list_t *fl; + + vec_validate_aligned (mm->per_thread_data, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + + fl = + vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + for (i = 0; i < tm->n_vlib_mains; i++) + { + memif_per_thread_data_t *ptd = + vec_elt_at_index (mm->per_thread_data, i); + vlib_buffer_t *bt = &ptd->buffer_template; + vlib_buffer_init_for_free_list (bt, fl); + bt->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; + bt->total_length_not_including_first_buffer = 0; + vnet_buffer (bt)->sw_if_index[VLIB_TX] = (u32) ~ 0; + + /* initially prealloc copy_ops so we can use + _vec_len instead of vec_elen */ + vec_validate_aligned (ptd->copy_ops, 0, CLIB_CACHE_LINE_BYTES); + vec_reset_length (ptd->copy_ops); + vec_validate_aligned (ptd->buffers, 0, CLIB_CACHE_LINE_BYTES); + vec_reset_length (ptd->buffers); + } + } pool_get (mm->interfaces, mif); memset (mif, 0, sizeof (*mif)); - mif->key = args->key; - mif->if_index = mif - mm->interfaces; + mif->dev_instance = mif - mm->interfaces; + mif->socket_file_index = msf - mm->socket_files; + mif->id = args->id; mif->sw_if_index = mif->hw_if_index = mif->per_interface_next_index = ~0; - mif->listener_index = ~0; - mif->connection.index = mif->interrupt_line.index = ~0; - mif->connection.fd = mif->interrupt_line.fd = -1; + mif->mode = args->mode; + if (args->secret) + mif->secret = vec_dup (args->secret); if (tm->n_vlib_mains > 1) + clib_spinlock_init (&mif->lockp); + + if (mif->mode == MEMIF_INTERFACE_MODE_ETHERNET) { - mif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, - CLIB_CACHE_LINE_BYTES); - memset ((void *) mif->lockp, 0, CLIB_CACHE_LINE_BYTES); - } - if (!args->hw_addr_set) + if (!args->hw_addr_set) + { + f64 now = vlib_time_now (vm); + u32 rnd; + rnd = (u32) (now * 1e6); + rnd = random_u32 (&rnd); + + memcpy (args->hw_addr + 2, &rnd, sizeof (rnd)); + args->hw_addr[0] = 2; + args->hw_addr[1] = 0xfe; + } + error = ethernet_register_interface (vnm, memif_device_class.index, + mif->dev_instance, args->hw_addr, + &mif->hw_if_index, + memif_eth_flag_change); + } + else if (mif->mode == MEMIF_INTERFACE_MODE_IP) { - f64 now = vlib_time_now (vm); - u32 rnd; - rnd = (u32) (now * 1e6); - rnd = random_u32 (&rnd); - - memcpy (args->hw_addr + 2, &rnd, sizeof (rnd)); - args->hw_addr[0] = 2; - args->hw_addr[1] = 0xfe; + mif->hw_if_index = + vnet_register_interface (vnm, memif_device_class.index, + mif->dev_instance, + memif_ip_hw_if_class.index, + mif->dev_instance); } - - error = ethernet_register_interface (vnm, memif_device_class.index, - mif->if_index, args->hw_addr, - &mif->hw_if_index, - memif_eth_flag_change); + else + error = clib_error_return (0, "unsupported interface mode"); if (error) { - clib_error_report (error); - ret = VNET_API_ERROR_SYSCALL_ERROR_1; + ret = VNET_API_ERROR_SYSCALL_ERROR_2; goto error; } sw = vnet_get_hw_sw_interface (vnm, mif->hw_if_index); mif->sw_if_index = sw->sw_if_index; - mif->log2_ring_size = args->log2_ring_size; - mif->buffer_size = args->buffer_size; - - /* TODO: make configurable */ - mif->num_s2m_rings = 1; - mif->num_m2s_rings = 1; - - mhash_set_mem (&mm->if_index_by_key, &args->key, &mif->if_index, 0); - - if (args->socket_filename != 0) - mif->socket_filename = args->socket_filename; - else - mif->socket_filename = vec_dup (mm->default_socket_filename); + mif->cfg.log2_ring_size = args->log2_ring_size; + mif->cfg.buffer_size = args->buffer_size; + mif->cfg.num_s2m_rings = + args->is_master ? args->rx_queues : args->tx_queues; + mif->cfg.num_m2s_rings = + args->is_master ? args->tx_queues : args->rx_queues; args->sw_if_index = mif->sw_if_index; - if (args->is_master) + /* If this is new one, start listening */ + if (msf->is_listener && msf->ref_cnt == 0) { - struct sockaddr_un un = { 0 }; struct stat file_stat; - int on = 1; - memif_listener_t *listener = 0; + clib_socket_t *s = clib_mem_alloc (sizeof (clib_socket_t)); - if (stat ((char *) mif->socket_filename, &file_stat) == 0) - { - if (!S_ISSOCK (file_stat.st_mode)) - { - errno = ENOTSOCK; - ret = VNET_API_ERROR_SYSCALL_ERROR_2; - goto error; - } - /* *INDENT-OFF* */ - pool_foreach (listener, mm->listeners, - ({ - if (listener->sock_dev == file_stat.st_dev && - listener->sock_ino == file_stat.st_ino) - { - /* attach memif to the existing listener */ - mif->listener_index = listener->index; - ++listener->usage_counter; - goto signal; - } - })); - /* *INDENT-ON* */ - unlink ((char *) mif->socket_filename); - } + ASSERT (msf->sock == 0); + msf->sock = s; - pool_get (mm->listeners, listener); - memset (listener, 0, sizeof (*listener)); - listener->socket.fd = -1; - listener->socket.index = ~0; - listener->index = listener - mm->listeners; - listener->usage_counter = 1; + memset (s, 0, sizeof (clib_socket_t)); + s->config = (char *) msf->filename; + s->flags = CLIB_SOCKET_F_IS_SERVER | + CLIB_SOCKET_F_ALLOW_GROUP_WRITE | + CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED; - if ((listener->socket.fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_3; - goto error; - } - - un.sun_family = AF_UNIX; - strncpy ((char *) un.sun_path, (char *) mif->socket_filename, - sizeof (un.sun_path) - 1); - - if (setsockopt (listener->socket.fd, SOL_SOCKET, SO_PASSCRED, - &on, sizeof (on)) < 0) + if ((error = clib_socket_init (s))) { ret = VNET_API_ERROR_SYSCALL_ERROR_4; goto error; } - if (bind (listener->socket.fd, (struct sockaddr *) &un, - sizeof (un)) == -1) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_5; - goto error; - } - if (listen (listener->socket.fd, 1) == -1) - { - ret = VNET_API_ERROR_SYSCALL_ERROR_6; - goto error; - } - if (stat ((char *) mif->socket_filename, &file_stat) == -1) + if (stat ((char *) msf->filename, &file_stat) == -1) { - ret = VNET_API_ERROR_SYSCALL_ERROR_7; + ret = VNET_API_ERROR_SYSCALL_ERROR_8; goto error; } - listener->sock_dev = file_stat.st_dev; - listener->sock_ino = file_stat.st_ino; - - unix_file_t template = { 0 }; + clib_file_t template = { 0 }; template.read_function = memif_conn_fd_accept_ready; - template.file_descriptor = listener->socket.fd; - template.private_data = listener->index; - listener->socket.index = unix_file_add (&unix_main, &template); - - mif->listener_index = listener->index; + template.file_descriptor = msf->sock->fd; + template.private_data = mif->socket_file_index; + template.description = format (0, "memif listener %s", msf->filename); + memif_file_add (&msf->sock->private_data, &template); } - else + + msf->ref_cnt++; + + if (args->is_master == 0) { mif->flags |= MEMIF_IF_FLAG_IS_SLAVE; + if (args->is_zero_copy) + mif->flags |= MEMIF_IF_FLAG_ZERO_COPY; } -#if 0 - /* use configured or generate random MAC address */ - if (!args->hw_addr_set && - tm->n_vlib_mains > 1 && pool_elts (mm->interfaces) == 1) - memif_worker_thread_enable (); -#endif + hw = vnet_get_hw_interface (vnm, mif->hw_if_index); + hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE; + vnet_hw_interface_set_input_node (vnm, mif->hw_if_index, + memif_input_node.index); + + mhash_set (&msf->dev_instance_by_id, &mif->id, mif->dev_instance, 0); -signal: if (pool_elts (mm->interfaces) == 1) { vlib_process_signal_event (vm, memif_process_node.index, MEMIF_PROCESS_EVENT_START, 0); } - return 0; + goto done; error: if (mif->hw_if_index != ~0) { - ethernet_delete_interface (vnm, mif->hw_if_index); + if (mif->mode == MEMIF_INTERFACE_MODE_IP) + vnet_delete_hw_interface (vnm, mif->hw_if_index); + else + ethernet_delete_interface (vnm, mif->hw_if_index); mif->hw_if_index = ~0; } - memif_close_if (mm, mif); - return ret; -} - -int -memif_delete_if (vlib_main_t * vm, u64 key) -{ - vnet_main_t *vnm = vnet_get_main (); - memif_main_t *mm = &memif_main; - memif_if_t *mif; - uword *p; - - p = mhash_get (&mm->if_index_by_key, &key); - if (p == NULL) - { - clib_warning ("Memory interface with key 0x%" PRIx64 " does not exist", - key); - return VNET_API_ERROR_SYSCALL_ERROR_1; - } - mif = pool_elt_at_index (mm->interfaces, p[0]); - mif->flags |= MEMIF_IF_FLAG_DELETING; - - /* bring down the interface */ - vnet_hw_interface_set_flags (vnm, mif->hw_if_index, 0); - vnet_sw_interface_set_flags (vnm, mif->sw_if_index, 0); - - /* remove the interface */ - ethernet_delete_interface (vnm, mif->hw_if_index); - mif->hw_if_index = ~0; - memif_close_if (mm, mif); - - if (pool_elts (mm->interfaces) == 0) + memif_delete_if (vm, mif); + if (error) { - vlib_process_signal_event (vm, memif_process_node.index, - MEMIF_PROCESS_EVENT_STOP, 0); + memif_log_err (mif, "%U", format_clib_error, error); + clib_error_free (error); } + return ret; -#if 0 - if (tm->n_vlib_mains > 1 && pool_elts (mm->interfaces) == 0) - memif_worker_thread_disable (); -#endif - - return 0; +done: + return rv; } static clib_error_t * memif_init (vlib_main_t * vm) { memif_main_t *mm = &memif_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - vlib_thread_registration_t *tr; - uword *p; memset (mm, 0, sizeof (memif_main_t)); - mm->input_cpu_first_index = 0; - mm->input_cpu_count = 1; + mm->log_class = vlib_log_register_class ("memif_plugin", 0); + memif_log_debug (0, "initialized"); /* initialize binary API */ memif_plugin_api_hookup (vm); - /* find out which cpus will be used for input */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - tr = p ? (vlib_thread_registration_t *) p[0] : 0; - - if (tr && tr->count > 0) - { - mm->input_cpu_first_index = tr->first_index; - mm->input_cpu_count = tr->count; - } - - mhash_init (&mm->if_index_by_key, sizeof (uword), sizeof (u64)); - - vec_validate_aligned (mm->rx_buffers, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - /* set default socket filename */ - vec_validate (mm->default_socket_filename, - strlen (MEMIF_DEFAULT_SOCKET_FILENAME)); - strncpy ((char *) mm->default_socket_filename, - MEMIF_DEFAULT_SOCKET_FILENAME, - vec_len (mm->default_socket_filename) - 1); + /* + * Pre-stuff socket filename pool with a non-modifieable mapping + * for socket-id 0 to MEMIF_DEFAULT_SOCKET_FILENAME in the + * default run-time directory. + */ + memif_socket_filename_add_del (1, 0, (u8 *) MEMIF_DEFAULT_SOCKET_FILENAME); return 0; } @@ -1032,7 +1040,7 @@ VLIB_INIT_FUNCTION (memif_init); /* *INDENT-OFF* */ VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER, - .description = "Packet Memory Interface (experimetal)", + .description = "Packet Memory Interface (experimental)", }; /* *INDENT-ON* */