examples/vhost/virtio_net.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <stdbool.h>
  36 #include <linux/virtio_net.h>
  37
  38 #include <rte_mbuf.h>
  39 #include <rte_memcpy.h>
  40 #include <rte_vhost.h>
  41
  42 #include "main.h"
  43
  44 /*
  45  * A very simple vhost-user net driver implementation, without
  46  * any extra features being enabled, such as TSO and mrg-Rx.
  47  */
  48
  49 void
  50 vs_vhost_net_setup(struct vhost_dev *dev)
  51 {
  52         uint16_t i;
  53         int vid = dev->vid;
  54         struct vhost_queue *queue;
  55
  56         RTE_LOG(INFO, VHOST_CONFIG,
  57                 "setting builtin vhost-user net driver\n");
  58
  59         rte_vhost_get_negotiated_features(vid, &dev->features);
  60         if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))
  61                 dev->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
  62         else
  63                 dev->hdr_len = sizeof(struct virtio_net_hdr);
  64
  65         rte_vhost_get_mem_table(vid, &dev->mem);
  66
  67         dev->nr_vrings = rte_vhost_get_vring_num(vid);
  68         for (i = 0; i < dev->nr_vrings; i++) {
  69                 queue = &dev->queues[i];
  70
  71                 queue->last_used_idx  = 0;
  72                 queue->last_avail_idx = 0;
  73                 rte_vhost_get_vhost_vring(vid, i, &queue->vr);
  74         }
  75 }
  76
  77 void
  78 vs_vhost_net_remove(struct vhost_dev *dev)
  79 {
  80         free(dev->mem);
  81 }
  82
  83 static __rte_always_inline int
  84 enqueue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
  85             struct rte_mbuf *m, uint16_t desc_idx)
  86 {
  87         uint32_t desc_avail, desc_offset;
  88         uint64_t desc_chunck_len;
  89         uint32_t mbuf_avail, mbuf_offset;
  90         uint32_t cpy_len;
  91         struct vring_desc *desc;
  92         uint64_t desc_addr, desc_gaddr;
  93         struct virtio_net_hdr virtio_hdr = {0, 0, 0, 0, 0, 0};
  94         /* A counter to avoid desc dead loop chain */
  95         uint16_t nr_desc = 1;
  96
  97         desc = &vr->desc[desc_idx];
  98         desc_chunck_len = desc->len;
  99         desc_gaddr = desc->addr;
 100         desc_addr = rte_vhost_va_from_guest_pa(
 101                         dev->mem, desc_gaddr, &desc_chunck_len);
 102         /*
 103          * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
 104          * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
 105          * otherwise stores offset on the stack instead of in a register.
 106          */
 107         if (unlikely(desc->len < dev->hdr_len) || !desc_addr)
 108                 return -1;
 109
 110         rte_prefetch0((void *)(uintptr_t)desc_addr);
 111
 112         /* write virtio-net header */
 113         if (likely(desc_chunck_len >= dev->hdr_len)) {
 114                 *(struct virtio_net_hdr *)(uintptr_t)desc_addr = virtio_hdr;
 115                 desc_offset = dev->hdr_len;
 116         } else {
 117                 uint64_t len;
 118                 uint64_t remain = dev->hdr_len;
 119                 uint64_t src = (uint64_t)(uintptr_t)&virtio_hdr, dst;
 120                 uint64_t guest_addr = desc_gaddr;
 121
 122                 while (remain) {
 123                         len = remain;
 124                         dst = rte_vhost_va_from_guest_pa(dev->mem,
 125                                         guest_addr, &len);
 126                         if (unlikely(!dst || !len))
 127                                 return -1;
 128
 129                         rte_memcpy((void *)(uintptr_t)dst,
 130                                         (void *)(uintptr_t)src,
 131                                         len);
 132
 133                         remain -= len;
 134                         guest_addr += len;
 135                         dst += len;
 136                 }
 137
 138                 desc_chunck_len = desc->len - dev->hdr_len;
 139                 desc_gaddr += dev->hdr_len;
 140                 desc_addr = rte_vhost_va_from_guest_pa(
 141                                 dev->mem, desc_gaddr,
 142                                 &desc_chunck_len);
 143                 if (unlikely(!desc_addr))
 144                         return -1;
 145
 146                 desc_offset = 0;
 147         }
 148
 149         desc_avail  = desc->len - dev->hdr_len;
 150
 151         mbuf_avail  = rte_pktmbuf_data_len(m);
 152         mbuf_offset = 0;
 153         while (mbuf_avail != 0 || m->next != NULL) {
 154                 /* done with current mbuf, fetch next */
 155                 if (mbuf_avail == 0) {
 156                         m = m->next;
 157
 158                         mbuf_offset = 0;
 159                         mbuf_avail  = rte_pktmbuf_data_len(m);
 160                 }
 161
 162                 /* done with current desc buf, fetch next */
 163                 if (desc_avail == 0) {
 164                         if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
 165                                 /* Room in vring buffer is not enough */
 166                                 return -1;
 167                         }
 168                         if (unlikely(desc->next >= vr->size ||
 169                                      ++nr_desc > vr->size))
 170                                 return -1;
 171
 172                         desc = &vr->desc[desc->next];
 173                         desc_chunck_len = desc->len;
 174                         desc_gaddr = desc->addr;
 175                         desc_addr = rte_vhost_va_from_guest_pa(
 176                                         dev->mem, desc_gaddr, &desc_chunck_len);
 177                         if (unlikely(!desc_addr))
 178                                 return -1;
 179
 180                         desc_offset = 0;
 181                         desc_avail  = desc->len;
 182                 } else if (unlikely(desc_chunck_len == 0)) {
 183                         desc_chunck_len = desc_avail;
 184                         desc_gaddr += desc_offset;
 185                         desc_addr = rte_vhost_va_from_guest_pa(dev->mem,
 186                                         desc_gaddr,
 187                                         &desc_chunck_len);
 188                         if (unlikely(!desc_addr))
 189                                 return -1;
 190
 191                         desc_offset = 0;
 192                 }
 193
 194                 cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
 195                 rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
 196                         rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 197                         cpy_len);
 198
 199                 mbuf_avail  -= cpy_len;
 200                 mbuf_offset += cpy_len;
 201                 desc_avail  -= cpy_len;
 202                 desc_offset += cpy_len;
 203                 desc_chunck_len -= cpy_len;
 204         }
 205
 206         return 0;
 207 }
 208
 209 uint16_t
 210 vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 211                 struct rte_mbuf **pkts, uint32_t count)
 212 {
 213         struct vhost_queue *queue;
 214         struct rte_vhost_vring *vr;
 215         uint16_t avail_idx, free_entries, start_idx;
 216         uint16_t desc_indexes[MAX_PKT_BURST];
 217         uint16_t used_idx;
 218         uint32_t i;
 219
 220         queue = &dev->queues[queue_id];
 221         vr    = &queue->vr;
 222
 223         avail_idx = *((volatile uint16_t *)&vr->avail->idx);
 224         start_idx = queue->last_used_idx;
 225         free_entries = avail_idx - start_idx;
 226         count = RTE_MIN(count, free_entries);
 227         count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
 228         if (count == 0)
 229                 return 0;
 230
 231         /* Retrieve all of the desc indexes first to avoid caching issues. */
 232         rte_prefetch0(&vr->avail->ring[start_idx & (vr->size - 1)]);
 233         for (i = 0; i < count; i++) {
 234                 used_idx = (start_idx + i) & (vr->size - 1);
 235                 desc_indexes[i] = vr->avail->ring[used_idx];
 236                 vr->used->ring[used_idx].id = desc_indexes[i];
 237                 vr->used->ring[used_idx].len = pkts[i]->pkt_len +
 238                                                dev->hdr_len;
 239         }
 240
 241         rte_prefetch0(&vr->desc[desc_indexes[0]]);
 242         for (i = 0; i < count; i++) {
 243                 uint16_t desc_idx = desc_indexes[i];
 244                 int err;
 245
 246                 err = enqueue_pkt(dev, vr, pkts[i], desc_idx);
 247                 if (unlikely(err)) {
 248                         used_idx = (start_idx + i) & (vr->size - 1);
 249                         vr->used->ring[used_idx].len = dev->hdr_len;
 250                 }
 251
 252                 if (i + 1 < count)
 253                         rte_prefetch0(&vr->desc[desc_indexes[i+1]]);
 254         }
 255
 256         rte_smp_wmb();
 257
 258         *(volatile uint16_t *)&vr->used->idx += count;
 259         queue->last_used_idx += count;
 260
 261         /* flush used->idx update before we read avail->flags. */
 262         rte_mb();
 263
 264         /* Kick the guest if necessary. */
 265         if (!(vr->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
 266                         && (vr->callfd >= 0))
 267                 eventfd_write(vr->callfd, (eventfd_t)1);
 268         return count;
 269 }
 270
 271 static __rte_always_inline int
 272 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 273             struct rte_mbuf *m, uint16_t desc_idx,
 274             struct rte_mempool *mbuf_pool)
 275 {
 276         struct vring_desc *desc;
 277         uint64_t desc_addr, desc_gaddr;
 278         uint32_t desc_avail, desc_offset;
 279         uint64_t desc_chunck_len;
 280         uint32_t mbuf_avail, mbuf_offset;
 281         uint32_t cpy_len;
 282         struct rte_mbuf *cur = m, *prev = m;
 283         /* A counter to avoid desc dead loop chain */
 284         uint32_t nr_desc = 1;
 285
 286         desc = &vr->desc[desc_idx];
 287         if (unlikely((desc->len < dev->hdr_len)) ||
 288                         (desc->flags & VRING_DESC_F_INDIRECT))
 289                 return -1;
 290
 291         desc_chunck_len = desc->len;
 292         desc_gaddr = desc->addr;
 293         desc_addr = rte_vhost_va_from_guest_pa(
 294                         dev->mem, desc_gaddr, &desc_chunck_len);
 295         if (unlikely(!desc_addr))
 296                 return -1;
 297
 298         /*
 299          * We don't support ANY_LAYOUT, neither VERSION_1, meaning
 300          * a Tx packet from guest must have 2 desc buffers at least:
 301          * the first for storing the header and the others for
 302          * storing the data.
 303          *
 304          * And since we don't support TSO, we could simply skip the
 305          * header.
 306          */
 307         desc = &vr->desc[desc->next];
 308         desc_chunck_len = desc->len;
 309         desc_gaddr = desc->addr;
 310         desc_addr = rte_vhost_va_from_guest_pa(
 311                         dev->mem, desc_gaddr, &desc_chunck_len);
 312         if (unlikely(!desc_addr))
 313                 return -1;
 314         rte_prefetch0((void *)(uintptr_t)desc_addr);
 315
 316         desc_offset = 0;
 317         desc_avail  = desc->len;
 318         nr_desc    += 1;
 319
 320         mbuf_offset = 0;
 321         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
 322         while (1) {
 323                 cpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);
 324                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
 325                                                    mbuf_offset),
 326                         (void *)((uintptr_t)(desc_addr + desc_offset)),
 327                         cpy_len);
 328
 329                 mbuf_avail  -= cpy_len;
 330                 mbuf_offset += cpy_len;
 331                 desc_avail  -= cpy_len;
 332                 desc_offset += cpy_len;
 333                 desc_chunck_len -= cpy_len;
 334
 335                 /* This desc reaches to its end, get the next one */
 336                 if (desc_avail == 0) {
 337                         if ((desc->flags & VRING_DESC_F_NEXT) == 0)
 338                                 break;
 339
 340                         if (unlikely(desc->next >= vr->size ||
 341                                      ++nr_desc > vr->size))
 342                                 return -1;
 343                         desc = &vr->desc[desc->next];
 344
 345                         desc_chunck_len = desc->len;
 346                         desc_gaddr = desc->addr;
 347                         desc_addr = rte_vhost_va_from_guest_pa(
 348                                         dev->mem, desc_gaddr, &desc_chunck_len);
 349                         if (unlikely(!desc_addr))
 350                                 return -1;
 351                         rte_prefetch0((void *)(uintptr_t)desc_addr);
 352
 353                         desc_offset = 0;
 354                         desc_avail  = desc->len;
 355                 } else if (unlikely(desc_chunck_len == 0)) {
 356                         desc_chunck_len = desc_avail;
 357                         desc_gaddr += desc_offset;
 358                         desc_addr = rte_vhost_va_from_guest_pa(dev->mem,
 359                                         desc_gaddr,
 360                                         &desc_chunck_len);
 361                         if (unlikely(!desc_addr))
 362                                 return -1;
 363
 364                         desc_offset = 0;
 365                 }
 366
 367                 /*
 368                  * This mbuf reaches to its end, get a new one
 369                  * to hold more data.
 370                  */
 371                 if (mbuf_avail == 0) {
 372                         cur = rte_pktmbuf_alloc(mbuf_pool);
 373                         if (unlikely(cur == NULL)) {
 374                                 RTE_LOG(ERR, VHOST_DATA, "Failed to "
 375                                         "allocate memory for mbuf.\n");
 376                                 return -1;
 377                         }
 378
 379                         prev->next = cur;
 380                         prev->data_len = mbuf_offset;
 381                         m->nb_segs += 1;
 382                         m->pkt_len += mbuf_offset;
 383                         prev = cur;
 384
 385                         mbuf_offset = 0;
 386                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
 387                 }
 388         }
 389
 390         prev->data_len = mbuf_offset;
 391         m->pkt_len    += mbuf_offset;
 392
 393         return 0;
 394 }
 395
 396 uint16_t
 397 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 398         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 399 {
 400         struct vhost_queue *queue;
 401         struct rte_vhost_vring *vr;
 402         uint32_t desc_indexes[MAX_PKT_BURST];
 403         uint32_t used_idx;
 404         uint32_t i = 0;
 405         uint16_t free_entries;
 406         uint16_t avail_idx;
 407
 408         queue = &dev->queues[queue_id];
 409         vr    = &queue->vr;
 410
 411         free_entries = *((volatile uint16_t *)&vr->avail->idx) -
 412                         queue->last_avail_idx;
 413         if (free_entries == 0)
 414                 return 0;
 415
 416         /* Prefetch available and used ring */
 417         avail_idx = queue->last_avail_idx & (vr->size - 1);
 418         used_idx  = queue->last_used_idx  & (vr->size - 1);
 419         rte_prefetch0(&vr->avail->ring[avail_idx]);
 420         rte_prefetch0(&vr->used->ring[used_idx]);
 421
 422         count = RTE_MIN(count, MAX_PKT_BURST);
 423         count = RTE_MIN(count, free_entries);
 424
 425         if (unlikely(count == 0))
 426                 return 0;
 427
 428         /*
 429          * Retrieve all of the head indexes first and pre-update used entries
 430          * to avoid caching issues.
 431          */
 432         for (i = 0; i < count; i++) {
 433                 avail_idx = (queue->last_avail_idx + i) & (vr->size - 1);
 434                 used_idx  = (queue->last_used_idx  + i) & (vr->size - 1);
 435                 desc_indexes[i] = vr->avail->ring[avail_idx];
 436
 437                 vr->used->ring[used_idx].id  = desc_indexes[i];
 438                 vr->used->ring[used_idx].len = 0;
 439         }
 440
 441         /* Prefetch descriptor index. */
 442         rte_prefetch0(&vr->desc[desc_indexes[0]]);
 443         for (i = 0; i < count; i++) {
 444                 int err;
 445
 446                 if (likely(i + 1 < count))
 447                         rte_prefetch0(&vr->desc[desc_indexes[i + 1]]);
 448
 449                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
 450                 if (unlikely(pkts[i] == NULL)) {
 451                         RTE_LOG(ERR, VHOST_DATA,
 452                                 "Failed to allocate memory for mbuf.\n");
 453                         break;
 454                 }
 455
 456                 err = dequeue_pkt(dev, vr, pkts[i], desc_indexes[i], mbuf_pool);
 457                 if (unlikely(err)) {
 458                         rte_pktmbuf_free(pkts[i]);
 459                         break;
 460                 }
 461
 462         }
 463
 464         queue->last_avail_idx += i;
 465         queue->last_used_idx += i;
 466         rte_smp_wmb();
 467         rte_smp_rmb();
 468
 469         vr->used->idx += i;
 470
 471         if (!(vr->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
 472                         && (vr->callfd >= 0))
 473                 eventfd_write(vr->callfd, (eventfd_t)1);
 474
 475         return i;
 476 }