drivers/net/virtio/virtio_user/vhost_kernel.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/types.h>
  35 #include <sys/stat.h>
  36 #include <fcntl.h>
  37 #include <unistd.h>
  38
  39 #include <rte_memory.h>
  40 #include <rte_eal_memconfig.h>
  41
  42 #include "vhost.h"
  43 #include "virtio_user_dev.h"
  44 #include "vhost_kernel_tap.h"
  45
  46 struct vhost_memory_kernel {
  47         uint32_t nregions;
  48         uint32_t padding;
  49         struct vhost_memory_region regions[0];
  50 };
  51
  52 /* vhost kernel ioctls */
  53 #define VHOST_VIRTIO 0xAF
  54 #define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, __u64)
  55 #define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, __u64)
  56 #define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01)
  57 #define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02)
  58 #define VHOST_SET_MEM_TABLE _IOW(VHOST_VIRTIO, 0x03, struct vhost_memory_kernel)
  59 #define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
  60 #define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
  61 #define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, struct vhost_vring_state)
  62 #define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, struct vhost_vring_addr)
  63 #define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
  64 #define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
  65 #define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, struct vhost_vring_file)
  66 #define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
  67 #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
  68 #define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, struct vhost_vring_file)
  69
  70 static uint64_t max_regions = 64;
  71
  72 static void
  73 get_vhost_kernel_max_regions(void)
  74 {
  75         int fd;
  76         char buf[20] = {'\0'};
  77
  78         fd = open("/sys/module/vhost/parameters/max_mem_regions", O_RDONLY);
  79         if (fd < 0)
  80                 return;
  81
  82         if (read(fd, buf, sizeof(buf) - 1) > 0)
  83                 max_regions = strtoull(buf, NULL, 10);
  84
  85         close(fd);
  86 }
  87
  88 static uint64_t vhost_req_user_to_kernel[] = {
  89         [VHOST_USER_SET_OWNER] = VHOST_SET_OWNER,
  90         [VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
  91         [VHOST_USER_SET_FEATURES] = VHOST_SET_FEATURES,
  92         [VHOST_USER_GET_FEATURES] = VHOST_GET_FEATURES,
  93         [VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
  94         [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
  95         [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
  96         [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
  97         [VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
  98         [VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
  99         [VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
 100 };
 101
 102 /* By default, vhost kernel module allows 64 regions, but DPDK allows
 103  * 256 segments. As a relief, below function merges those virtually
 104  * adjacent memsegs into one region.
 105  */
 106 static struct vhost_memory_kernel *
 107 prepare_vhost_memory_kernel(void)
 108 {
 109         uint32_t i, j, k = 0;
 110         struct rte_memseg *seg;
 111         struct vhost_memory_region *mr;
 112         struct vhost_memory_kernel *vm;
 113
 114         vm = malloc(sizeof(struct vhost_memory_kernel) +
 115                     max_regions *
 116                     sizeof(struct vhost_memory_region));
 117         if (!vm)
 118                 return NULL;
 119
 120         for (i = 0; i < RTE_MAX_MEMSEG; ++i) {
 121                 seg = &rte_eal_get_configuration()->mem_config->memseg[i];
 122                 if (!seg->addr)
 123                         break;
 124
 125                 int new_region = 1;
 126
 127                 for (j = 0; j < k; ++j) {
 128                         mr = &vm->regions[j];
 129
 130                         if (mr->userspace_addr + mr->memory_size ==
 131                             (uint64_t)(uintptr_t)seg->addr) {
 132                                 mr->memory_size += seg->len;
 133                                 new_region = 0;
 134                                 break;
 135                         }
 136
 137                         if ((uint64_t)(uintptr_t)seg->addr + seg->len ==
 138                             mr->userspace_addr) {
 139                                 mr->guest_phys_addr =
 140                                         (uint64_t)(uintptr_t)seg->addr;
 141                                 mr->userspace_addr =
 142                                         (uint64_t)(uintptr_t)seg->addr;
 143                                 mr->memory_size += seg->len;
 144                                 new_region = 0;
 145                                 break;
 146                         }
 147                 }
 148
 149                 if (new_region == 0)
 150                         continue;
 151
 152                 mr = &vm->regions[k++];
 153                 /* use vaddr here! */
 154                 mr->guest_phys_addr = (uint64_t)(uintptr_t)seg->addr;
 155                 mr->userspace_addr = (uint64_t)(uintptr_t)seg->addr;
 156                 mr->memory_size = seg->len;
 157                 mr->mmap_offset = 0;
 158
 159                 if (k >= max_regions) {
 160                         free(vm);
 161                         return NULL;
 162                 }
 163         }
 164
 165         vm->nregions = k;
 166         vm->padding = 0;
 167         return vm;
 168 }
 169
 170 /* with below features, vhost kernel does not need to do the checksum and TSO,
 171  * these info will be passed to virtio_user through virtio net header.
 172  */
 173 #define VHOST_KERNEL_GUEST_OFFLOADS_MASK        \
 174         ((1ULL << VIRTIO_NET_F_GUEST_CSUM) |    \
 175          (1ULL << VIRTIO_NET_F_GUEST_TSO4) |    \
 176          (1ULL << VIRTIO_NET_F_GUEST_TSO6) |    \
 177          (1ULL << VIRTIO_NET_F_GUEST_ECN)  |    \
 178          (1ULL << VIRTIO_NET_F_GUEST_UFO))
 179
 180 /* with below features, when flows from virtio_user to vhost kernel
 181  * (1) if flows goes up through the kernel networking stack, it does not need
 182  * to verify checksum, which can save CPU cycles;
 183  * (2) if flows goes through a Linux bridge and outside from an interface
 184  * (kernel driver), checksum and TSO will be done by GSO in kernel or even
 185  * offloaded into real physical device.
 186  */
 187 #define VHOST_KERNEL_HOST_OFFLOADS_MASK         \
 188         ((1ULL << VIRTIO_NET_F_HOST_TSO4) |     \
 189          (1ULL << VIRTIO_NET_F_HOST_TSO6) |     \
 190          (1ULL << VIRTIO_NET_F_CSUM))
 191
 192 static unsigned int
 193 tap_support_features(void)
 194 {
 195         int tapfd;
 196         unsigned int tap_features;
 197
 198         tapfd = open(PATH_NET_TUN, O_RDWR);
 199         if (tapfd < 0) {
 200                 PMD_DRV_LOG(ERR, "fail to open %s: %s",
 201                             PATH_NET_TUN, strerror(errno));
 202                 return -1;
 203         }
 204
 205         if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) {
 206                 PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno));
 207                 close(tapfd);
 208                 return -1;
 209         }
 210
 211         close(tapfd);
 212         return tap_features;
 213 }
 214
 215 static int
 216 vhost_kernel_ioctl(struct virtio_user_dev *dev,
 217                    enum vhost_user_request req,
 218                    void *arg)
 219 {
 220         int ret = -1;
 221         unsigned int i;
 222         uint64_t req_kernel;
 223         struct vhost_memory_kernel *vm = NULL;
 224         int vhostfd;
 225         unsigned int queue_sel;
 226         unsigned int features;
 227
 228         PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]);
 229
 230         req_kernel = vhost_req_user_to_kernel[req];
 231
 232         if (req_kernel == VHOST_SET_MEM_TABLE) {
 233                 vm = prepare_vhost_memory_kernel();
 234                 if (!vm)
 235                         return -1;
 236                 arg = (void *)vm;
 237         }
 238
 239         if (req_kernel == VHOST_SET_FEATURES) {
 240                 /* We don't need memory protection here */
 241                 *(uint64_t *)arg &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
 242
 243                 /* VHOST kernel does not know about below flags */
 244                 *(uint64_t *)arg &= ~VHOST_KERNEL_GUEST_OFFLOADS_MASK;
 245                 *(uint64_t *)arg &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK;
 246
 247                 *(uint64_t *)arg &= ~(1ULL << VIRTIO_NET_F_MQ);
 248         }
 249
 250         switch (req_kernel) {
 251         case VHOST_SET_VRING_NUM:
 252         case VHOST_SET_VRING_ADDR:
 253         case VHOST_SET_VRING_BASE:
 254         case VHOST_GET_VRING_BASE:
 255         case VHOST_SET_VRING_KICK:
 256         case VHOST_SET_VRING_CALL:
 257                 queue_sel = *(unsigned int *)arg;
 258                 vhostfd = dev->vhostfds[queue_sel / 2];
 259                 *(unsigned int *)arg = queue_sel % 2;
 260                 PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
 261                             vhostfd, *(unsigned int *)arg);
 262                 break;
 263         default:
 264                 vhostfd = -1;
 265         }
 266         if (vhostfd == -1) {
 267                 for (i = 0; i < dev->max_queue_pairs; ++i) {
 268                         if (dev->vhostfds[i] < 0)
 269                                 continue;
 270
 271                         ret = ioctl(dev->vhostfds[i], req_kernel, arg);
 272                         if (ret < 0)
 273                                 break;
 274                 }
 275         } else {
 276                 ret = ioctl(vhostfd, req_kernel, arg);
 277         }
 278
 279         if (!ret && req_kernel == VHOST_GET_FEATURES) {
 280                 features = tap_support_features();
 281                 /* with tap as the backend, all these features are supported
 282                  * but not claimed by vhost-net, so we add them back when
 283                  * reporting to upper layer.
 284                  */
 285                 if (features & IFF_VNET_HDR) {
 286                         *((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
 287                         *((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
 288                 }
 289
 290                 /* vhost_kernel will not declare this feature, but it does
 291                  * support multi-queue.
 292                  */
 293                 if (features & IFF_MULTI_QUEUE)
 294                         *(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ);
 295         }
 296
 297         if (vm)
 298                 free(vm);
 299
 300         if (ret < 0)
 301                 PMD_DRV_LOG(ERR, "%s failed: %s",
 302                             vhost_msg_strings[req], strerror(errno));
 303
 304         return ret;
 305 }
 306
 307 /**
 308  * Set up environment to talk with a vhost kernel backend.
 309  *
 310  * @return
 311  *   - (-1) if fail to set up;
 312  *   - (>=0) if successful.
 313  */
 314 static int
 315 vhost_kernel_setup(struct virtio_user_dev *dev)
 316 {
 317         int vhostfd;
 318         uint32_t i;
 319
 320         get_vhost_kernel_max_regions();
 321
 322         for (i = 0; i < dev->max_queue_pairs; ++i) {
 323                 vhostfd = open(dev->path, O_RDWR);
 324                 if (vhostfd < 0) {
 325                         PMD_DRV_LOG(ERR, "fail to open %s, %s",
 326                                     dev->path, strerror(errno));
 327                         return -1;
 328                 }
 329
 330                 dev->vhostfds[i] = vhostfd;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static int
 337 vhost_kernel_set_backend(int vhostfd, int tapfd)
 338 {
 339         struct vhost_vring_file f;
 340
 341         f.fd = tapfd;
 342         f.index = 0;
 343         if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
 344                 PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
 345                                 strerror(errno));
 346                 return -1;
 347         }
 348
 349         f.index = 1;
 350         if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
 351                 PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
 352                                 strerror(errno));
 353                 return -1;
 354         }
 355
 356         return 0;
 357 }
 358
 359 static int
 360 vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 361                                uint16_t pair_idx,
 362                                int enable)
 363 {
 364         int hdr_size;
 365         int vhostfd;
 366         int tapfd;
 367         int req_mq = (dev->max_queue_pairs > 1);
 368
 369         vhostfd = dev->vhostfds[pair_idx];
 370
 371         if (!enable) {
 372                 if (dev->tapfds[pair_idx] >= 0) {
 373                         close(dev->tapfds[pair_idx]);
 374                         dev->tapfds[pair_idx] = -1;
 375                 }
 376                 return vhost_kernel_set_backend(vhostfd, -1);
 377         } else if (dev->tapfds[pair_idx] >= 0) {
 378                 return 0;
 379         }
 380
 381         if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) ||
 382             (dev->features & (1ULL << VIRTIO_F_VERSION_1)))
 383                 hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 384         else
 385                 hdr_size = sizeof(struct virtio_net_hdr);
 386
 387         tapfd = vhost_kernel_open_tap(&dev->ifname, hdr_size, req_mq,
 388                          (char *)dev->mac_addr, dev->features);
 389         if (tapfd < 0) {
 390                 PMD_DRV_LOG(ERR, "fail to open tap for vhost kernel");
 391                 return -1;
 392         }
 393
 394         if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
 395                 PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
 396                 close(tapfd);
 397                 return -1;
 398         }
 399
 400         dev->tapfds[pair_idx] = tapfd;
 401         return 0;
 402 }
 403
 404 struct virtio_user_backend_ops ops_kernel = {
 405         .setup = vhost_kernel_setup,
 406         .send_request = vhost_kernel_ioctl,
 407         .enable_qp = vhost_kernel_enable_queue_pair
 408 };