X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;ds=sidebyside;f=lib%2Flibrte_vhost%2Fvhost_user.c;h=f4c7ce462f0ba6198a7bb8d24ab1fa059391748e;hb=055c52583a2794da8ba1e85a48cce3832372b12f;hp=ad2e8d3802f24ac36dd5d9dccad59bb4de7ea167;hpb=f239aed5e674965691846e8ce3f187dd47523689;p=deb_dpdk.git diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index ad2e8d38..f4c7ce46 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -48,6 +48,7 @@ #include #include +#include "iotlb.h" #include "vhost.h" #include "vhost_user.h" @@ -76,6 +77,8 @@ static const char *vhost_message_str[VHOST_USER_MAX] = { [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE", [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP", [VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU", + [VHOST_USER_SET_SLAVE_REQ_FD] = "VHOST_USER_SET_SLAVE_REQ_FD", + [VHOST_USER_IOTLB_MSG] = "VHOST_USER_IOTLB_MSG", }; static uint64_t @@ -122,6 +125,11 @@ vhost_backend_cleanup(struct virtio_net *dev) munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); dev->log_addr = 0; } + + if (dev->slave_req_fd >= 0) { + close(dev->slave_req_fd); + dev->slave_req_fd = -1; + } } /* @@ -230,6 +238,15 @@ vhost_user_set_vring_num(struct virtio_net *dev, return -1; } + vq->batch_copy_elems = rte_malloc(NULL, + vq->size * sizeof(struct batch_copy_elem), + RTE_CACHE_LINE_SIZE); + if (!vq->batch_copy_elems) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for batching copy.\n"); + return -1; + } + return 0; } @@ -297,6 +314,9 @@ out: dev->virtqueue[index] = vq; vhost_devices[dev->vid] = dev; + if (old_vq != vq) + vhost_user_iotlb_init(dev, index); + return dev; } #else @@ -307,10 +327,7 @@ numa_realloc(struct virtio_net *dev, int index __rte_unused) } #endif -/* - * Converts QEMU virtual address to Vhost virtual address. This function is - * used to convert the ring addresses to our address space. - */ +/* Converts QEMU virtual address to Vhost virtual address. */ static uint64_t qva_to_vva(struct virtio_net *dev, uint64_t qva) { @@ -331,50 +348,69 @@ qva_to_vva(struct virtio_net *dev, uint64_t qva) return 0; } + /* - * The virtio device sends us the desc, used and avail ring addresses. - * This function then converts these to our address space. + * Converts ring address to Vhost virtual address. + * If IOMMU is enabled, the ring address is a guest IO virtual address, + * else it is a QEMU virtual address. */ -static int -vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg) +static uint64_t +ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t ra, uint64_t size) { - struct vhost_virtqueue *vq; + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { + uint64_t vva; - if (dev->mem == NULL) - return -1; + vva = vhost_user_iotlb_cache_find(vq, ra, + &size, VHOST_ACCESS_RW); + if (!vva) + vhost_user_iotlb_miss(dev, ra, VHOST_ACCESS_RW); - /* addr->index refers to the queue index. The txq 1, rxq is 0. */ - vq = dev->virtqueue[msg->payload.addr.index]; + return vva; + } + + return qva_to_vva(dev, ra); +} + +static struct virtio_net * +translate_ring_addresses(struct virtio_net *dev, int vq_index) +{ + struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; + struct vhost_vring_addr *addr = &vq->ring_addrs; /* The addresses are converted from QEMU virtual to Vhost virtual. */ - vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev, - msg->payload.addr.desc_user_addr); + if (vq->desc && vq->avail && vq->used) + return dev; + + vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->desc_user_addr, sizeof(struct vring_desc)); if (vq->desc == 0) { - RTE_LOG(ERR, VHOST_CONFIG, + RTE_LOG(DEBUG, VHOST_CONFIG, "(%d) failed to find desc ring address.\n", dev->vid); - return -1; + return dev; } - dev = numa_realloc(dev, msg->payload.addr.index); - vq = dev->virtqueue[msg->payload.addr.index]; + dev = numa_realloc(dev, vq_index); + vq = dev->virtqueue[vq_index]; + addr = &vq->ring_addrs; - vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev, - msg->payload.addr.avail_user_addr); + vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->avail_user_addr, sizeof(struct vring_avail)); if (vq->avail == 0) { - RTE_LOG(ERR, VHOST_CONFIG, + RTE_LOG(DEBUG, VHOST_CONFIG, "(%d) failed to find avail ring address.\n", dev->vid); - return -1; + return dev; } - vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev, - msg->payload.addr.used_user_addr); + vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev, + vq, addr->used_user_addr, sizeof(struct vring_used)); if (vq->used == 0) { - RTE_LOG(ERR, VHOST_CONFIG, + RTE_LOG(DEBUG, VHOST_CONFIG, "(%d) failed to find used ring address.\n", dev->vid); - return -1; + return dev; } if (vq->last_used_idx != vq->used->idx) { @@ -386,7 +422,7 @@ vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg) vq->last_avail_idx = vq->used->idx; } - vq->log_guest_addr = msg->payload.addr.log_guest_addr; + vq->log_guest_addr = addr->log_guest_addr; LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n", dev->vid, vq->desc); @@ -397,6 +433,43 @@ vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg) LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n", dev->vid, vq->log_guest_addr); + return dev; +} + +/* + * The virtio device sends us the desc, used and avail ring addresses. + * This function then converts these to our address space. + */ +static int +vhost_user_set_vring_addr(struct virtio_net **pdev, VhostUserMsg *msg) +{ + struct vhost_virtqueue *vq; + struct vhost_vring_addr *addr = &msg->payload.addr; + struct virtio_net *dev = *pdev; + + if (dev->mem == NULL) + return -1; + + /* addr->index refers to the queue index. The txq 1, rxq is 0. */ + vq = dev->virtqueue[msg->payload.addr.index]; + + /* + * Rings addresses should not be interpreted as long as the ring is not + * started and enabled + */ + memcpy(&vq->ring_addrs, addr, sizeof(*addr)); + + vring_invalidate(dev, vq); + + if (vq->enabled && (dev->features & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { + dev = translate_ring_addresses(dev, msg->payload.state.index); + if (!dev) + return -1; + + *pdev = dev; + } + return 0; } @@ -453,7 +526,7 @@ add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, uint64_t host_phys_addr; uint64_t size; - host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)host_user_addr); + host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr); size = page_size - (guest_phys_addr & (page_size - 1)); size = RTE_MIN(size, reg_size); @@ -464,7 +537,7 @@ add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, while (reg_size > 0) { size = RTE_MIN(reg_size, page_size); - host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t) + host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t) host_user_addr); add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size); @@ -620,7 +693,7 @@ err_mmap: static int vq_is_ready(struct vhost_virtqueue *vq) { - return vq && vq->desc && + return vq && vq->desc && vq->avail && vq->used && vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD && vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD; } @@ -668,10 +741,11 @@ vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg) } static void -vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg) +vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *pmsg) { struct vhost_vring_file file; struct vhost_virtqueue *vq; + struct virtio_net *dev = *pdev; file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) @@ -681,7 +755,23 @@ vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg) RTE_LOG(INFO, VHOST_CONFIG, "vring kick idx:%d file:%d\n", file.index, file.fd); + /* Interpret ring addresses only when ring is started. */ + dev = translate_ring_addresses(dev, file.index); + if (!dev) + return; + + *pdev = dev; + vq = dev->virtqueue[file.index]; + + /* + * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated, + * the ring starts already enabled. Otherwise, it is enabled via + * the SET_VRING_ENABLE message. + */ + if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) + vq->enabled = 1; + if (vq->kickfd >= 0) close(vq->kickfd); vq->kickfd = file.fd; @@ -741,6 +831,9 @@ vhost_user_get_vring_base(struct virtio_net *dev, rte_free(vq->shadow_used_ring); vq->shadow_used_ring = NULL; + rte_free(vq->batch_copy_elems); + vq->batch_copy_elems = NULL; + return 0; } @@ -767,6 +860,27 @@ vhost_user_set_vring_enable(struct virtio_net *dev, return 0; } +static void +vhost_user_get_protocol_features(struct virtio_net *dev, + struct VhostUserMsg *msg) +{ + uint64_t features, protocol_features = VHOST_USER_PROTOCOL_FEATURES; + + rte_vhost_driver_get_features(dev->ifname, &features); + + /* + * REPLY_ACK protocol feature is only mandatory for now + * for IOMMU feature. If IOMMU is explicitly disabled by the + * application, disable also REPLY_ACK feature for older buggy + * Qemu versions (from v2.7.0 to v2.9.0). + */ + if (!(features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) + protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK); + + msg->payload.u64 = protocol_features; + msg->size = sizeof(msg->payload.u64); +} + static void vhost_user_set_protocol_features(struct virtio_net *dev, uint64_t protocol_features) @@ -874,6 +988,116 @@ vhost_user_net_set_mtu(struct virtio_net *dev, struct VhostUserMsg *msg) return 0; } +static int +vhost_user_set_req_fd(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + int fd = msg->fds[0]; + + if (fd < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Invalid file descriptor for slave channel (%d)\n", + fd); + return -1; + } + + dev->slave_req_fd = fd; + + return 0; +} + +static int +is_vring_iotlb_update(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg) +{ + struct vhost_vring_addr *ra; + uint64_t start, end; + + start = imsg->iova; + end = start + imsg->size; + + ra = &vq->ring_addrs; + if (ra->desc_user_addr >= start && ra->desc_user_addr < end) + return 1; + if (ra->avail_user_addr >= start && ra->avail_user_addr < end) + return 1; + if (ra->used_user_addr >= start && ra->used_user_addr < end) + return 1; + + return 0; +} + +static int +is_vring_iotlb_invalidate(struct vhost_virtqueue *vq, + struct vhost_iotlb_msg *imsg) +{ + uint64_t istart, iend, vstart, vend; + + istart = imsg->iova; + iend = istart + imsg->size - 1; + + vstart = (uintptr_t)vq->desc; + vend = vstart + sizeof(struct vring_desc) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + vstart = (uintptr_t)vq->avail; + vend = vstart + sizeof(struct vring_avail); + vend += sizeof(uint16_t) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + vstart = (uintptr_t)vq->used; + vend = vstart + sizeof(struct vring_used); + vend += sizeof(struct vring_used_elem) * vq->size - 1; + if (vstart <= iend && istart <= vend) + return 1; + + return 0; +} + +static int +vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg) +{ + struct virtio_net *dev = *pdev; + struct vhost_iotlb_msg *imsg = &msg->payload.iotlb; + uint16_t i; + uint64_t vva; + + switch (imsg->type) { + case VHOST_IOTLB_UPDATE: + vva = qva_to_vva(dev, imsg->uaddr); + if (!vva) + return -1; + + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_insert(vq, imsg->iova, vva, + imsg->size, imsg->perm); + + if (is_vring_iotlb_update(vq, imsg)) + *pdev = dev = translate_ring_addresses(dev, i); + } + break; + case VHOST_IOTLB_INVALIDATE: + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_remove(vq, imsg->iova, + imsg->size); + + if (is_vring_iotlb_invalidate(vq, imsg)) + vring_invalidate(dev, vq); + } + break; + default: + RTE_LOG(ERR, VHOST_CONFIG, "Invalid IOTLB message type (%d)\n", + imsg->type); + return -1; + } + + return 0; +} + /* return bytes# of read on success or negative val on failure. */ static int read_vhost_message(int sockfd, struct VhostUserMsg *msg) @@ -907,8 +1131,16 @@ read_vhost_message(int sockfd, struct VhostUserMsg *msg) static int send_vhost_message(int sockfd, struct VhostUserMsg *msg) { - int ret; + if (!msg) + return 0; + + return send_fd_message(sockfd, (char *)msg, + VHOST_USER_HDR_SIZE + msg->size, NULL, 0); +} +static int +send_vhost_reply(int sockfd, struct VhostUserMsg *msg) +{ if (!msg) return 0; @@ -917,10 +1149,7 @@ send_vhost_message(int sockfd, struct VhostUserMsg *msg) msg->flags |= VHOST_USER_VERSION; msg->flags |= VHOST_USER_REPLY_MASK; - ret = send_fd_message(sockfd, (char *)msg, - VHOST_USER_HDR_SIZE + msg->size, NULL, 0); - - return ret; + return send_vhost_message(sockfd, msg); } /* @@ -931,7 +1160,7 @@ vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev, VhostUserMsg *msg) { uint16_t vring_idx; - switch (msg->request) { + switch (msg->request.master) { case VHOST_USER_SET_VRING_KICK: case VHOST_USER_SET_VRING_CALL: case VHOST_USER_SET_VRING_ERR: @@ -983,7 +1212,7 @@ vhost_user_msg_handler(int vid, int fd) } ret = read_vhost_message(fd, &msg); - if (ret <= 0 || msg.request >= VHOST_USER_MAX) { + if (ret <= 0 || msg.request.master >= VHOST_USER_MAX) { if (ret < 0) RTE_LOG(ERR, VHOST_CONFIG, "vhost read message failed\n"); @@ -998,8 +1227,12 @@ vhost_user_msg_handler(int vid, int fd) } ret = 0; - RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n", - vhost_message_str[msg.request]); + if (msg.request.master != VHOST_USER_IOTLB_MSG) + RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n", + vhost_message_str[msg.request.master]); + else + RTE_LOG(DEBUG, VHOST_CONFIG, "read message %s\n", + vhost_message_str[msg.request.master]); ret = vhost_user_check_and_alloc_queue_pair(dev, &msg); if (ret < 0) { @@ -1008,20 +1241,19 @@ vhost_user_msg_handler(int vid, int fd) return -1; } - switch (msg.request) { + switch (msg.request.master) { case VHOST_USER_GET_FEATURES: msg.payload.u64 = vhost_user_get_features(dev); msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_FEATURES: vhost_user_set_features(dev, msg.payload.u64); break; case VHOST_USER_GET_PROTOCOL_FEATURES: - msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES; - msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + vhost_user_get_protocol_features(dev, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_PROTOCOL_FEATURES: vhost_user_set_protocol_features(dev, msg.payload.u64); @@ -1043,7 +1275,7 @@ vhost_user_msg_handler(int vid, int fd) /* it needs a reply */ msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_LOG_FD: close(msg.fds[0]); @@ -1054,7 +1286,7 @@ vhost_user_msg_handler(int vid, int fd) vhost_user_set_vring_num(dev, &msg); break; case VHOST_USER_SET_VRING_ADDR: - vhost_user_set_vring_addr(dev, &msg); + vhost_user_set_vring_addr(&dev, &msg); break; case VHOST_USER_SET_VRING_BASE: vhost_user_set_vring_base(dev, &msg); @@ -1063,11 +1295,11 @@ vhost_user_msg_handler(int vid, int fd) case VHOST_USER_GET_VRING_BASE: vhost_user_get_vring_base(dev, &msg); msg.size = sizeof(msg.payload.state); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_VRING_KICK: - vhost_user_set_vring_kick(dev, &msg); + vhost_user_set_vring_kick(&dev, &msg); break; case VHOST_USER_SET_VRING_CALL: vhost_user_set_vring_call(dev, &msg); @@ -1082,7 +1314,7 @@ vhost_user_msg_handler(int vid, int fd) case VHOST_USER_GET_QUEUE_NUM: msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS; msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_VRING_ENABLE: @@ -1096,6 +1328,14 @@ vhost_user_msg_handler(int vid, int fd) ret = vhost_user_net_set_mtu(dev, &msg); break; + case VHOST_USER_SET_SLAVE_REQ_FD: + ret = vhost_user_set_req_fd(dev, &msg); + break; + + case VHOST_USER_IOTLB_MSG: + ret = vhost_user_iotlb_msg(&dev, &msg); + break; + default: ret = -1; break; @@ -1105,7 +1345,7 @@ vhost_user_msg_handler(int vid, int fd) if (msg.flags & VHOST_USER_NEED_REPLY) { msg.payload.u64 = !!ret; msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); } if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) { @@ -1124,3 +1364,29 @@ vhost_user_msg_handler(int vid, int fd) return 0; } + +int +vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm) +{ + int ret; + struct VhostUserMsg msg = { + .request.slave = VHOST_USER_SLAVE_IOTLB_MSG, + .flags = VHOST_USER_VERSION, + .size = sizeof(msg.payload.iotlb), + .payload.iotlb = { + .iova = iova, + .perm = perm, + .type = VHOST_IOTLB_MISS, + }, + }; + + ret = send_vhost_message(dev->slave_req_fd, &msg); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to send IOTLB miss message (%d)\n", + ret); + return ret; + } + + return 0; +}