lib/librte_vhost/virtio_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <linux/virtio_net.h>
   8
   9 #include <rte_mbuf.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_ether.h>
  12 #include <rte_ip.h>
  13 #include <rte_vhost.h>
  14 #include <rte_tcp.h>
  15 #include <rte_udp.h>
  16 #include <rte_sctp.h>
  17 #include <rte_arp.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_malloc.h>
  20
  21 #include "iotlb.h"
  22 #include "vhost.h"
  23
  24 #define MAX_PKT_BURST 32
  25
  26 #define MAX_BATCH_LEN 256
  27
  28 static  __rte_always_inline bool
  29 rxvq_is_mergeable(struct virtio_net *dev)
  30 {
  31         return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF);
  32 }
  33
  34 static bool
  35 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring)
  36 {
  37         return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring;
  38 }
  39
  40 static __rte_always_inline void *
  41 alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq,
  42                 uint64_t desc_addr, uint64_t desc_len)
  43 {
  44         void *idesc;
  45         uint64_t src, dst;
  46         uint64_t len, remain = desc_len;
  47
  48         idesc = rte_malloc(__func__, desc_len, 0);
  49         if (unlikely(!idesc))
  50                 return 0;
  51
  52         dst = (uint64_t)(uintptr_t)idesc;
  53
  54         while (remain) {
  55                 len = remain;
  56                 src = vhost_iova_to_vva(dev, vq, desc_addr, &len,
  57                                 VHOST_ACCESS_RO);
  58                 if (unlikely(!src || !len)) {
  59                         rte_free(idesc);
  60                         return 0;
  61                 }
  62
  63                 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len);
  64
  65                 remain -= len;
  66                 dst += len;
  67                 desc_addr += len;
  68         }
  69
  70         return idesc;
  71 }
  72
  73 static __rte_always_inline void
  74 free_ind_table(void *idesc)
  75 {
  76         rte_free(idesc);
  77 }
  78
  79 static __rte_always_inline void
  80 do_flush_shadow_used_ring_split(struct virtio_net *dev,
  81                         struct vhost_virtqueue *vq,
  82                         uint16_t to, uint16_t from, uint16_t size)
  83 {
  84         rte_memcpy(&vq->used->ring[to],
  85                         &vq->shadow_used_split[from],
  86                         size * sizeof(struct vring_used_elem));
  87         vhost_log_cache_used_vring(dev, vq,
  88                         offsetof(struct vring_used, ring[to]),
  89                         size * sizeof(struct vring_used_elem));
  90 }
  91
  92 static __rte_always_inline void
  93 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
  94 {
  95         uint16_t used_idx = vq->last_used_idx & (vq->size - 1);
  96
  97         if (used_idx + vq->shadow_used_idx <= vq->size) {
  98                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0,
  99                                           vq->shadow_used_idx);
 100         } else {
 101                 uint16_t size;
 102
 103                 /* update used ring interval [used_idx, vq->size] */
 104                 size = vq->size - used_idx;
 105                 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);
 106
 107                 /* update the left half used ring interval [0, left_size] */
 108                 do_flush_shadow_used_ring_split(dev, vq, 0, size,
 109                                           vq->shadow_used_idx - size);
 110         }
 111         vq->last_used_idx += vq->shadow_used_idx;
 112
 113         rte_smp_wmb();
 114
 115         vhost_log_cache_sync(dev, vq);
 116
 117         *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx;
 118         vq->shadow_used_idx = 0;
 119         vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
 120                 sizeof(vq->used->idx));
 121 }
 122
 123 static __rte_always_inline void
 124 update_shadow_used_ring_split(struct vhost_virtqueue *vq,
 125                          uint16_t desc_idx, uint32_t len)
 126 {
 127         uint16_t i = vq->shadow_used_idx++;
 128
 129         vq->shadow_used_split[i].id  = desc_idx;
 130         vq->shadow_used_split[i].len = len;
 131 }
 132
 133 static __rte_always_inline void
 134 flush_shadow_used_ring_packed(struct virtio_net *dev,
 135                         struct vhost_virtqueue *vq)
 136 {
 137         int i;
 138         uint16_t used_idx = vq->last_used_idx;
 139
 140         /* Split loop in two to save memory barriers */
 141         for (i = 0; i < vq->shadow_used_idx; i++) {
 142                 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id;
 143                 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len;
 144
 145                 used_idx += vq->shadow_used_packed[i].count;
 146                 if (used_idx >= vq->size)
 147                         used_idx -= vq->size;
 148         }
 149
 150         rte_smp_wmb();
 151
 152         for (i = 0; i < vq->shadow_used_idx; i++) {
 153                 uint16_t flags;
 154
 155                 if (vq->shadow_used_packed[i].len)
 156                         flags = VRING_DESC_F_WRITE;
 157                 else
 158                         flags = 0;
 159
 160                 if (vq->used_wrap_counter) {
 161                         flags |= VRING_DESC_F_USED;
 162                         flags |= VRING_DESC_F_AVAIL;
 163                 } else {
 164                         flags &= ~VRING_DESC_F_USED;
 165                         flags &= ~VRING_DESC_F_AVAIL;
 166                 }
 167
 168                 vq->desc_packed[vq->last_used_idx].flags = flags;
 169
 170                 vhost_log_cache_used_vring(dev, vq,
 171                                         vq->last_used_idx *
 172                                         sizeof(struct vring_packed_desc),
 173                                         sizeof(struct vring_packed_desc));
 174
 175                 vq->last_used_idx += vq->shadow_used_packed[i].count;
 176                 if (vq->last_used_idx >= vq->size) {
 177                         vq->used_wrap_counter ^= 1;
 178                         vq->last_used_idx -= vq->size;
 179                 }
 180         }
 181
 182         rte_smp_wmb();
 183         vq->shadow_used_idx = 0;
 184         vhost_log_cache_sync(dev, vq);
 185 }
 186
 187 static __rte_always_inline void
 188 update_shadow_used_ring_packed(struct vhost_virtqueue *vq,
 189                          uint16_t desc_idx, uint32_t len, uint16_t count)
 190 {
 191         uint16_t i = vq->shadow_used_idx++;
 192
 193         vq->shadow_used_packed[i].id  = desc_idx;
 194         vq->shadow_used_packed[i].len = len;
 195         vq->shadow_used_packed[i].count = count;
 196 }
 197
 198 static inline void
 199 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 200 {
 201         struct batch_copy_elem *elem = vq->batch_copy_elems;
 202         uint16_t count = vq->batch_copy_nb_elems;
 203         int i;
 204
 205         for (i = 0; i < count; i++) {
 206                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 207                 vhost_log_cache_write(dev, vq, elem[i].log_addr, elem[i].len);
 208                 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
 209         }
 210
 211         vq->batch_copy_nb_elems = 0;
 212 }
 213
 214 static inline void
 215 do_data_copy_dequeue(struct vhost_virtqueue *vq)
 216 {
 217         struct batch_copy_elem *elem = vq->batch_copy_elems;
 218         uint16_t count = vq->batch_copy_nb_elems;
 219         int i;
 220
 221         for (i = 0; i < count; i++)
 222                 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
 223
 224         vq->batch_copy_nb_elems = 0;
 225 }
 226
 227 /* avoid write operation when necessary, to lessen cache issues */
 228 #define ASSIGN_UNLESS_EQUAL(var, val) do {      \
 229         if ((var) != (val))                     \
 230                 (var) = (val);                  \
 231 } while (0)
 232
 233 static __rte_always_inline void
 234 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 235 {
 236         uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK;
 237
 238         if (m_buf->ol_flags & PKT_TX_TCP_SEG)
 239                 csum_l4 |= PKT_TX_TCP_CKSUM;
 240
 241         if (csum_l4) {
 242                 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
 243                 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len;
 244
 245                 switch (csum_l4) {
 246                 case PKT_TX_TCP_CKSUM:
 247                         net_hdr->csum_offset = (offsetof(struct tcp_hdr,
 248                                                 cksum));
 249                         break;
 250                 case PKT_TX_UDP_CKSUM:
 251                         net_hdr->csum_offset = (offsetof(struct udp_hdr,
 252                                                 dgram_cksum));
 253                         break;
 254                 case PKT_TX_SCTP_CKSUM:
 255                         net_hdr->csum_offset = (offsetof(struct sctp_hdr,
 256                                                 cksum));
 257                         break;
 258                 }
 259         } else {
 260                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0);
 261                 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0);
 262                 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0);
 263         }
 264
 265         /* IP cksum verification cannot be bypassed, then calculate here */
 266         if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
 267                 struct ipv4_hdr *ipv4_hdr;
 268
 269                 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct ipv4_hdr *,
 270                                                    m_buf->l2_len);
 271                 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
 272         }
 273
 274         if (m_buf->ol_flags & PKT_TX_TCP_SEG) {
 275                 if (m_buf->ol_flags & PKT_TX_IPV4)
 276                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
 277                 else
 278                         net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
 279                 net_hdr->gso_size = m_buf->tso_segsz;
 280                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len
 281                                         + m_buf->l4_len;
 282         } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) {
 283                 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
 284                 net_hdr->gso_size = m_buf->tso_segsz;
 285                 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len +
 286                         m_buf->l4_len;
 287         } else {
 288                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0);
 289                 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0);
 290                 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0);
 291         }
 292 }
 293
 294 static __rte_always_inline int
 295 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 296                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 297                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 298 {
 299         uint16_t vec_id = *vec_idx;
 300
 301         while (desc_len) {
 302                 uint64_t desc_addr;
 303                 uint64_t desc_chunck_len = desc_len;
 304
 305                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 306                         return -1;
 307
 308                 desc_addr = vhost_iova_to_vva(dev, vq,
 309                                 desc_iova,
 310                                 &desc_chunck_len,
 311                                 perm);
 312                 if (unlikely(!desc_addr))
 313                         return -1;
 314
 315                 buf_vec[vec_id].buf_iova = desc_iova;
 316                 buf_vec[vec_id].buf_addr = desc_addr;
 317                 buf_vec[vec_id].buf_len  = desc_chunck_len;
 318
 319                 desc_len -= desc_chunck_len;
 320                 desc_iova += desc_chunck_len;
 321                 vec_id++;
 322         }
 323         *vec_idx = vec_id;
 324
 325         return 0;
 326 }
 327
 328 static __rte_always_inline int
 329 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 330                          uint32_t avail_idx, uint16_t *vec_idx,
 331                          struct buf_vector *buf_vec, uint16_t *desc_chain_head,
 332                          uint32_t *desc_chain_len, uint8_t perm)
 333 {
 334         uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
 335         uint16_t vec_id = *vec_idx;
 336         uint32_t len    = 0;
 337         uint64_t dlen;
 338         uint32_t nr_descs = vq->size;
 339         uint32_t cnt    = 0;
 340         struct vring_desc *descs = vq->desc;
 341         struct vring_desc *idesc = NULL;
 342
 343         if (unlikely(idx >= vq->size))
 344                 return -1;
 345
 346         *desc_chain_head = idx;
 347
 348         if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
 349                 dlen = vq->desc[idx].len;
 350                 nr_descs = dlen / sizeof(struct vring_desc);
 351                 if (unlikely(nr_descs > vq->size))
 352                         return -1;
 353
 354                 descs = (struct vring_desc *)(uintptr_t)
 355                         vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
 356                                                 &dlen,
 357                                                 VHOST_ACCESS_RO);
 358                 if (unlikely(!descs))
 359                         return -1;
 360
 361                 if (unlikely(dlen < vq->desc[idx].len)) {
 362                         /*
 363                          * The indirect desc table is not contiguous
 364                          * in process VA space, we have to copy it.
 365                          */
 366                         idesc = alloc_copy_ind_table(dev, vq,
 367                                         vq->desc[idx].addr, vq->desc[idx].len);
 368                         if (unlikely(!idesc))
 369                                 return -1;
 370
 371                         descs = idesc;
 372                 }
 373
 374                 idx = 0;
 375         }
 376
 377         while (1) {
 378                 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
 379                         free_ind_table(idesc);
 380                         return -1;
 381                 }
 382
 383                 len += descs[idx].len;
 384
 385                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 386                                                 descs[idx].addr, descs[idx].len,
 387                                                 perm))) {
 388                         free_ind_table(idesc);
 389                         return -1;
 390                 }
 391
 392                 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
 393                         break;
 394
 395                 idx = descs[idx].next;
 396         }
 397
 398         *desc_chain_len = len;
 399         *vec_idx = vec_id;
 400
 401         if (unlikely(!!idesc))
 402                 free_ind_table(idesc);
 403
 404         return 0;
 405 }
 406
 407 /*
 408  * Returns -1 on fail, 0 on success
 409  */
 410 static inline int
 411 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 412                                 uint32_t size, struct buf_vector *buf_vec,
 413                                 uint16_t *num_buffers, uint16_t avail_head,
 414                                 uint16_t *nr_vec)
 415 {
 416         uint16_t cur_idx;
 417         uint16_t vec_idx = 0;
 418         uint16_t max_tries, tries = 0;
 419
 420         uint16_t head_idx = 0;
 421         uint32_t len = 0;
 422
 423         *num_buffers = 0;
 424         cur_idx  = vq->last_avail_idx;
 425
 426         if (rxvq_is_mergeable(dev))
 427                 max_tries = vq->size - 1;
 428         else
 429                 max_tries = 1;
 430
 431         while (size > 0) {
 432                 if (unlikely(cur_idx == avail_head))
 433                         return -1;
 434                 /*
 435                  * if we tried all available ring items, and still
 436                  * can't get enough buf, it means something abnormal
 437                  * happened.
 438                  */
 439                 if (unlikely(++tries > max_tries))
 440                         return -1;
 441
 442                 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx,
 443                                                 &vec_idx, buf_vec,
 444                                                 &head_idx, &len,
 445                                                 VHOST_ACCESS_RW) < 0))
 446                         return -1;
 447                 len = RTE_MIN(len, size);
 448                 update_shadow_used_ring_split(vq, head_idx, len);
 449                 size -= len;
 450
 451                 cur_idx++;
 452                 *num_buffers += 1;
 453         }
 454
 455         *nr_vec = vec_idx;
 456
 457         return 0;
 458 }
 459
 460 static __rte_always_inline int
 461 fill_vec_buf_packed_indirect(struct virtio_net *dev,
 462                         struct vhost_virtqueue *vq,
 463                         struct vring_packed_desc *desc, uint16_t *vec_idx,
 464                         struct buf_vector *buf_vec, uint32_t *len, uint8_t perm)
 465 {
 466         uint16_t i;
 467         uint32_t nr_descs;
 468         uint16_t vec_id = *vec_idx;
 469         uint64_t dlen;
 470         struct vring_packed_desc *descs, *idescs = NULL;
 471
 472         dlen = desc->len;
 473         descs = (struct vring_packed_desc *)(uintptr_t)
 474                 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO);
 475         if (unlikely(!descs))
 476                 return -1;
 477
 478         if (unlikely(dlen < desc->len)) {
 479                 /*
 480                  * The indirect desc table is not contiguous
 481                  * in process VA space, we have to copy it.
 482                  */
 483                 idescs = alloc_copy_ind_table(dev, vq, desc->addr, desc->len);
 484                 if (unlikely(!idescs))
 485                         return -1;
 486
 487                 descs = idescs;
 488         }
 489
 490         nr_descs =  desc->len / sizeof(struct vring_packed_desc);
 491         if (unlikely(nr_descs >= vq->size)) {
 492                 free_ind_table(idescs);
 493                 return -1;
 494         }
 495
 496         for (i = 0; i < nr_descs; i++) {
 497                 if (unlikely(vec_id >= BUF_VECTOR_MAX)) {
 498                         free_ind_table(idescs);
 499                         return -1;
 500                 }
 501
 502                 *len += descs[i].len;
 503                 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 504                                                 descs[i].addr, descs[i].len,
 505                                                 perm)))
 506                         return -1;
 507         }
 508         *vec_idx = vec_id;
 509
 510         if (unlikely(!!idescs))
 511                 free_ind_table(idescs);
 512
 513         return 0;
 514 }
 515
 516 static __rte_always_inline int
 517 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 518                                 uint16_t avail_idx, uint16_t *desc_count,
 519                                 struct buf_vector *buf_vec, uint16_t *vec_idx,
 520                                 uint16_t *buf_id, uint32_t *len, uint8_t perm)
 521 {
 522         bool wrap_counter = vq->avail_wrap_counter;
 523         struct vring_packed_desc *descs = vq->desc_packed;
 524         uint16_t vec_id = *vec_idx;
 525
 526         if (avail_idx < vq->last_avail_idx)
 527                 wrap_counter ^= 1;
 528
 529         if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
 530                 return -1;
 531
 532         /*
 533          * The ordering between desc flags and desc
 534          * content reads need to be enforced.
 535          */
 536         rte_smp_rmb();
 537
 538         *desc_count = 0;
 539         *len = 0;
 540
 541         while (1) {
 542                 if (unlikely(vec_id >= BUF_VECTOR_MAX))
 543                         return -1;
 544
 545                 if (unlikely(*desc_count >= vq->size))
 546                         return -1;
 547
 548                 *desc_count += 1;
 549                 *buf_id = descs[avail_idx].id;
 550
 551                 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) {
 552                         if (unlikely(fill_vec_buf_packed_indirect(dev, vq,
 553                                                         &descs[avail_idx],
 554                                                         &vec_id, buf_vec,
 555                                                         len, perm) < 0))
 556                                 return -1;
 557                 } else {
 558                         *len += descs[avail_idx].len;
 559
 560                         if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
 561                                                         descs[avail_idx].addr,
 562                                                         descs[avail_idx].len,
 563                                                         perm)))
 564                                 return -1;
 565                 }
 566
 567                 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0)
 568                         break;
 569
 570                 if (++avail_idx >= vq->size) {
 571                         avail_idx -= vq->size;
 572                         wrap_counter ^= 1;
 573                 }
 574         }
 575
 576         *vec_idx = vec_id;
 577
 578         return 0;
 579 }
 580
 581 /*
 582  * Returns -1 on fail, 0 on success
 583  */
 584 static inline int
 585 reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 586                                 uint32_t size, struct buf_vector *buf_vec,
 587                                 uint16_t *nr_vec, uint16_t *num_buffers,
 588                                 uint16_t *nr_descs)
 589 {
 590         uint16_t avail_idx;
 591         uint16_t vec_idx = 0;
 592         uint16_t max_tries, tries = 0;
 593
 594         uint16_t buf_id = 0;
 595         uint32_t len = 0;
 596         uint16_t desc_count;
 597
 598         *num_buffers = 0;
 599         avail_idx = vq->last_avail_idx;
 600
 601         if (rxvq_is_mergeable(dev))
 602                 max_tries = vq->size - 1;
 603         else
 604                 max_tries = 1;
 605
 606         while (size > 0) {
 607                 /*
 608                  * if we tried all available ring items, and still
 609                  * can't get enough buf, it means something abnormal
 610                  * happened.
 611                  */
 612                 if (unlikely(++tries > max_tries))
 613                         return -1;
 614
 615                 if (unlikely(fill_vec_buf_packed(dev, vq,
 616                                                 avail_idx, &desc_count,
 617                                                 buf_vec, &vec_idx,
 618                                                 &buf_id, &len,
 619                                                 VHOST_ACCESS_RW) < 0))
 620                         return -1;
 621
 622                 len = RTE_MIN(len, size);
 623                 update_shadow_used_ring_packed(vq, buf_id, len, desc_count);
 624                 size -= len;
 625
 626                 avail_idx += desc_count;
 627                 if (avail_idx >= vq->size)
 628                         avail_idx -= vq->size;
 629
 630                 *nr_descs += desc_count;
 631                 *num_buffers += 1;
 632         }
 633
 634         *nr_vec = vec_idx;
 635
 636         return 0;
 637 }
 638
 639 static __rte_always_inline int
 640 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 641                             struct rte_mbuf *m, struct buf_vector *buf_vec,
 642                             uint16_t nr_vec, uint16_t num_buffers)
 643 {
 644         uint32_t vec_idx = 0;
 645         uint32_t mbuf_offset, mbuf_avail;
 646         uint32_t buf_offset, buf_avail;
 647         uint64_t buf_addr, buf_iova, buf_len;
 648         uint32_t cpy_len;
 649         uint64_t hdr_addr;
 650         struct rte_mbuf *hdr_mbuf;
 651         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 652         struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;
 653         int error = 0;
 654
 655         if (unlikely(m == NULL)) {
 656                 error = -1;
 657                 goto out;
 658         }
 659
 660         buf_addr = buf_vec[vec_idx].buf_addr;
 661         buf_iova = buf_vec[vec_idx].buf_iova;
 662         buf_len = buf_vec[vec_idx].buf_len;
 663
 664         if (nr_vec > 1)
 665                 rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr);
 666
 667         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
 668                 error = -1;
 669                 goto out;
 670         }
 671
 672         hdr_mbuf = m;
 673         hdr_addr = buf_addr;
 674         if (unlikely(buf_len < dev->vhost_hlen))
 675                 hdr = &tmp_hdr;
 676         else
 677                 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 678
 679         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
 680                 dev->vid, num_buffers);
 681
 682         if (unlikely(buf_len < dev->vhost_hlen)) {
 683                 buf_offset = dev->vhost_hlen - buf_len;
 684                 vec_idx++;
 685                 buf_addr = buf_vec[vec_idx].buf_addr;
 686                 buf_iova = buf_vec[vec_idx].buf_iova;
 687                 buf_len = buf_vec[vec_idx].buf_len;
 688                 buf_avail = buf_len - buf_offset;
 689         } else {
 690                 buf_offset = dev->vhost_hlen;
 691                 buf_avail = buf_len - dev->vhost_hlen;
 692         }
 693
 694         mbuf_avail  = rte_pktmbuf_data_len(m);
 695         mbuf_offset = 0;
 696         while (mbuf_avail != 0 || m->next != NULL) {
 697                 /* done with current buf, get the next one */
 698                 if (buf_avail == 0) {
 699                         vec_idx++;
 700                         if (unlikely(vec_idx >= nr_vec)) {
 701                                 error = -1;
 702                                 goto out;
 703                         }
 704
 705                         buf_addr = buf_vec[vec_idx].buf_addr;
 706                         buf_iova = buf_vec[vec_idx].buf_iova;
 707                         buf_len = buf_vec[vec_idx].buf_len;
 708
 709                         /* Prefetch next buffer address. */
 710                         if (vec_idx + 1 < nr_vec)
 711                                 rte_prefetch0((void *)(uintptr_t)
 712                                                 buf_vec[vec_idx + 1].buf_addr);
 713                         buf_offset = 0;
 714                         buf_avail  = buf_len;
 715                 }
 716
 717                 /* done with current mbuf, get the next one */
 718                 if (mbuf_avail == 0) {
 719                         m = m->next;
 720
 721                         mbuf_offset = 0;
 722                         mbuf_avail  = rte_pktmbuf_data_len(m);
 723                 }
 724
 725                 if (hdr_addr) {
 726                         virtio_enqueue_offload(hdr_mbuf, &hdr->hdr);
 727                         if (rxvq_is_mergeable(dev))
 728                                 ASSIGN_UNLESS_EQUAL(hdr->num_buffers,
 729                                                 num_buffers);
 730
 731                         if (unlikely(hdr == &tmp_hdr)) {
 732                                 uint64_t len;
 733                                 uint64_t remain = dev->vhost_hlen;
 734                                 uint64_t src = (uint64_t)(uintptr_t)hdr, dst;
 735                                 uint64_t iova = buf_vec[0].buf_iova;
 736                                 uint16_t hdr_vec_idx = 0;
 737
 738                                 while (remain) {
 739                                         len = RTE_MIN(remain,
 740                                                 buf_vec[hdr_vec_idx].buf_len);
 741                                         dst = buf_vec[hdr_vec_idx].buf_addr;
 742                                         rte_memcpy((void *)(uintptr_t)dst,
 743                                                         (void *)(uintptr_t)src,
 744                                                         len);
 745
 746                                         PRINT_PACKET(dev, (uintptr_t)dst,
 747                                                         (uint32_t)len, 0);
 748                                         vhost_log_cache_write(dev, vq,
 749                                                         iova, len);
 750
 751                                         remain -= len;
 752                                         iova += len;
 753                                         src += len;
 754                                         hdr_vec_idx++;
 755                                 }
 756                         } else {
 757                                 PRINT_PACKET(dev, (uintptr_t)hdr_addr,
 758                                                 dev->vhost_hlen, 0);
 759                                 vhost_log_cache_write(dev, vq,
 760                                                 buf_vec[0].buf_iova,
 761                                                 dev->vhost_hlen);
 762                         }
 763
 764                         hdr_addr = 0;
 765                 }
 766
 767                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 768
 769                 if (likely(cpy_len > MAX_BATCH_LEN ||
 770                                         vq->batch_copy_nb_elems >= vq->size)) {
 771                         rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),
 772                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 773                                 cpy_len);
 774                         vhost_log_cache_write(dev, vq, buf_iova + buf_offset,
 775                                         cpy_len);
 776                         PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),
 777                                 cpy_len, 0);
 778                 } else {
 779                         batch_copy[vq->batch_copy_nb_elems].dst =
 780                                 (void *)((uintptr_t)(buf_addr + buf_offset));
 781                         batch_copy[vq->batch_copy_nb_elems].src =
 782                                 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
 783                         batch_copy[vq->batch_copy_nb_elems].log_addr =
 784                                 buf_iova + buf_offset;
 785                         batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 786                         vq->batch_copy_nb_elems++;
 787                 }
 788
 789                 mbuf_avail  -= cpy_len;
 790                 mbuf_offset += cpy_len;
 791                 buf_avail  -= cpy_len;
 792                 buf_offset += cpy_len;
 793         }
 794
 795 out:
 796
 797         return error;
 798 }
 799
 800 static __rte_always_inline uint32_t
 801 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 802         struct rte_mbuf **pkts, uint32_t count)
 803 {
 804         uint32_t pkt_idx = 0;
 805         uint16_t num_buffers;
 806         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 807         uint16_t avail_head;
 808
 809         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 810         avail_head = *((volatile uint16_t *)&vq->avail->idx);
 811
 812         /*
 813          * The ordering between avail index and
 814          * desc reads needs to be enforced.
 815          */
 816         rte_smp_rmb();
 817
 818         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 819                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 820                 uint16_t nr_vec = 0;
 821
 822                 if (unlikely(reserve_avail_buf_split(dev, vq,
 823                                                 pkt_len, buf_vec, &num_buffers,
 824                                                 avail_head, &nr_vec) < 0)) {
 825                         VHOST_LOG_DEBUG(VHOST_DATA,
 826                                 "(%d) failed to get enough desc from vring\n",
 827                                 dev->vid);
 828                         vq->shadow_used_idx -= num_buffers;
 829                         break;
 830                 }
 831
 832                 rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
 833
 834                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 835                         dev->vid, vq->last_avail_idx,
 836                         vq->last_avail_idx + num_buffers);
 837
 838                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
 839                                                 buf_vec, nr_vec,
 840                                                 num_buffers) < 0) {
 841                         vq->shadow_used_idx -= num_buffers;
 842                         break;
 843                 }
 844
 845                 vq->last_avail_idx += num_buffers;
 846         }
 847
 848         do_data_copy_enqueue(dev, vq);
 849
 850         if (likely(vq->shadow_used_idx)) {
 851                 flush_shadow_used_ring_split(dev, vq);
 852                 vhost_vring_call_split(dev, vq);
 853         }
 854
 855         return pkt_idx;
 856 }
 857
 858 static __rte_always_inline uint32_t
 859 virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 860         struct rte_mbuf **pkts, uint32_t count)
 861 {
 862         uint32_t pkt_idx = 0;
 863         uint16_t num_buffers;
 864         struct buf_vector buf_vec[BUF_VECTOR_MAX];
 865
 866         for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
 867                 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
 868                 uint16_t nr_vec = 0;
 869                 uint16_t nr_descs = 0;
 870
 871                 if (unlikely(reserve_avail_buf_packed(dev, vq,
 872                                                 pkt_len, buf_vec, &nr_vec,
 873                                                 &num_buffers, &nr_descs) < 0)) {
 874                         VHOST_LOG_DEBUG(VHOST_DATA,
 875                                 "(%d) failed to get enough desc from vring\n",
 876                                 dev->vid);
 877                         vq->shadow_used_idx -= num_buffers;
 878                         break;
 879                 }
 880
 881                 rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
 882
 883                 VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
 884                         dev->vid, vq->last_avail_idx,
 885                         vq->last_avail_idx + num_buffers);
 886
 887                 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],
 888                                                 buf_vec, nr_vec,
 889                                                 num_buffers) < 0) {
 890                         vq->shadow_used_idx -= num_buffers;
 891                         break;
 892                 }
 893
 894                 vq->last_avail_idx += nr_descs;
 895                 if (vq->last_avail_idx >= vq->size) {
 896                         vq->last_avail_idx -= vq->size;
 897                         vq->avail_wrap_counter ^= 1;
 898                 }
 899         }
 900
 901         do_data_copy_enqueue(dev, vq);
 902
 903         if (likely(vq->shadow_used_idx)) {
 904                 flush_shadow_used_ring_packed(dev, vq);
 905                 vhost_vring_call_packed(dev, vq);
 906         }
 907
 908         return pkt_idx;
 909 }
 910
 911 static __rte_always_inline uint32_t
 912 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
 913         struct rte_mbuf **pkts, uint32_t count)
 914 {
 915         struct vhost_virtqueue *vq;
 916         uint32_t nb_tx = 0;
 917
 918         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
 919         if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
 920                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
 921                         dev->vid, __func__, queue_id);
 922                 return 0;
 923         }
 924
 925         vq = dev->virtqueue[queue_id];
 926
 927         rte_spinlock_lock(&vq->access_lock);
 928
 929         if (unlikely(vq->enabled == 0))
 930                 goto out_access_unlock;
 931
 932         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 933                 vhost_user_iotlb_rd_lock(vq);
 934
 935         if (unlikely(vq->access_ok == 0))
 936                 if (unlikely(vring_translate(dev, vq) < 0))
 937                         goto out;
 938
 939         count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
 940         if (count == 0)
 941                 goto out;
 942
 943         if (vq_is_packed(dev))
 944                 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count);
 945         else
 946                 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count);
 947
 948 out:
 949         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
 950                 vhost_user_iotlb_rd_unlock(vq);
 951
 952 out_access_unlock:
 953         rte_spinlock_unlock(&vq->access_lock);
 954
 955         return nb_tx;
 956 }
 957
 958 uint16_t
 959 rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 960         struct rte_mbuf **pkts, uint16_t count)
 961 {
 962         struct virtio_net *dev = get_device(vid);
 963
 964         if (!dev)
 965                 return 0;
 966
 967         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
 968                 RTE_LOG(ERR, VHOST_DATA,
 969                         "(%d) %s: built-in vhost net backend is disabled.\n",
 970                         dev->vid, __func__);
 971                 return 0;
 972         }
 973
 974         return virtio_dev_rx(dev, queue_id, pkts, count);
 975 }
 976
 977 static inline bool
 978 virtio_net_with_host_offload(struct virtio_net *dev)
 979 {
 980         if (dev->features &
 981                         ((1ULL << VIRTIO_NET_F_CSUM) |
 982                          (1ULL << VIRTIO_NET_F_HOST_ECN) |
 983                          (1ULL << VIRTIO_NET_F_HOST_TSO4) |
 984                          (1ULL << VIRTIO_NET_F_HOST_TSO6) |
 985                          (1ULL << VIRTIO_NET_F_HOST_UFO)))
 986                 return true;
 987
 988         return false;
 989 }
 990
 991 static void
 992 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
 993 {
 994         struct ipv4_hdr *ipv4_hdr;
 995         struct ipv6_hdr *ipv6_hdr;
 996         void *l3_hdr = NULL;
 997         struct ether_hdr *eth_hdr;
 998         uint16_t ethertype;
 999
1000         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1001
1002         m->l2_len = sizeof(struct ether_hdr);
1003         ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
1004
1005         if (ethertype == ETHER_TYPE_VLAN) {
1006                 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
1007
1008                 m->l2_len += sizeof(struct vlan_hdr);
1009                 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
1010         }
1011
1012         l3_hdr = (char *)eth_hdr + m->l2_len;
1013
1014         switch (ethertype) {
1015         case ETHER_TYPE_IPv4:
1016                 ipv4_hdr = l3_hdr;
1017                 *l4_proto = ipv4_hdr->next_proto_id;
1018                 m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
1019                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1020                 m->ol_flags |= PKT_TX_IPV4;
1021                 break;
1022         case ETHER_TYPE_IPv6:
1023                 ipv6_hdr = l3_hdr;
1024                 *l4_proto = ipv6_hdr->proto;
1025                 m->l3_len = sizeof(struct ipv6_hdr);
1026                 *l4_hdr = (char *)l3_hdr + m->l3_len;
1027                 m->ol_flags |= PKT_TX_IPV6;
1028                 break;
1029         default:
1030                 m->l3_len = 0;
1031                 *l4_proto = 0;
1032                 *l4_hdr = NULL;
1033                 break;
1034         }
1035 }
1036
1037 static __rte_always_inline void
1038 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
1039 {
1040         uint16_t l4_proto = 0;
1041         void *l4_hdr = NULL;
1042         struct tcp_hdr *tcp_hdr = NULL;
1043
1044         if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
1045                 return;
1046
1047         parse_ethernet(m, &l4_proto, &l4_hdr);
1048         if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1049                 if (hdr->csum_start == (m->l2_len + m->l3_len)) {
1050                         switch (hdr->csum_offset) {
1051                         case (offsetof(struct tcp_hdr, cksum)):
1052                                 if (l4_proto == IPPROTO_TCP)
1053                                         m->ol_flags |= PKT_TX_TCP_CKSUM;
1054                                 break;
1055                         case (offsetof(struct udp_hdr, dgram_cksum)):
1056                                 if (l4_proto == IPPROTO_UDP)
1057                                         m->ol_flags |= PKT_TX_UDP_CKSUM;
1058                                 break;
1059                         case (offsetof(struct sctp_hdr, cksum)):
1060                                 if (l4_proto == IPPROTO_SCTP)
1061                                         m->ol_flags |= PKT_TX_SCTP_CKSUM;
1062                                 break;
1063                         default:
1064                                 break;
1065                         }
1066                 }
1067         }
1068
1069         if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1070                 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1071                 case VIRTIO_NET_HDR_GSO_TCPV4:
1072                 case VIRTIO_NET_HDR_GSO_TCPV6:
1073                         tcp_hdr = l4_hdr;
1074                         m->ol_flags |= PKT_TX_TCP_SEG;
1075                         m->tso_segsz = hdr->gso_size;
1076                         m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
1077                         break;
1078                 case VIRTIO_NET_HDR_GSO_UDP:
1079                         m->ol_flags |= PKT_TX_UDP_SEG;
1080                         m->tso_segsz = hdr->gso_size;
1081                         m->l4_len = sizeof(struct udp_hdr);
1082                         break;
1083                 default:
1084                         RTE_LOG(WARNING, VHOST_DATA,
1085                                 "unsupported gso type %u.\n", hdr->gso_type);
1086                         break;
1087                 }
1088         }
1089 }
1090
1091 static __rte_always_inline int
1092 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
1093                   struct buf_vector *buf_vec, uint16_t nr_vec,
1094                   struct rte_mbuf *m, struct rte_mempool *mbuf_pool)
1095 {
1096         uint32_t buf_avail, buf_offset;
1097         uint64_t buf_addr, buf_iova, buf_len;
1098         uint32_t mbuf_avail, mbuf_offset;
1099         uint32_t cpy_len;
1100         struct rte_mbuf *cur = m, *prev = m;
1101         struct virtio_net_hdr tmp_hdr;
1102         struct virtio_net_hdr *hdr = NULL;
1103         /* A counter to avoid desc dead loop chain */
1104         uint16_t vec_idx = 0;
1105         struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
1106         int error = 0;
1107
1108         buf_addr = buf_vec[vec_idx].buf_addr;
1109         buf_iova = buf_vec[vec_idx].buf_iova;
1110         buf_len = buf_vec[vec_idx].buf_len;
1111
1112         if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
1113                 error = -1;
1114                 goto out;
1115         }
1116
1117         if (likely(nr_vec > 1))
1118                 rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr);
1119
1120         if (virtio_net_with_host_offload(dev)) {
1121                 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
1122                         uint64_t len;
1123                         uint64_t remain = sizeof(struct virtio_net_hdr);
1124                         uint64_t src;
1125                         uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr;
1126                         uint16_t hdr_vec_idx = 0;
1127
1128                         /*
1129                          * No luck, the virtio-net header doesn't fit
1130                          * in a contiguous virtual area.
1131                          */
1132                         while (remain) {
1133                                 len = RTE_MIN(remain,
1134                                         buf_vec[hdr_vec_idx].buf_len);
1135                                 src = buf_vec[hdr_vec_idx].buf_addr;
1136                                 rte_memcpy((void *)(uintptr_t)dst,
1137                                                    (void *)(uintptr_t)src, len);
1138
1139                                 remain -= len;
1140                                 dst += len;
1141                                 hdr_vec_idx++;
1142                         }
1143
1144                         hdr = &tmp_hdr;
1145                 } else {
1146                         hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);
1147                         rte_prefetch0(hdr);
1148                 }
1149         }
1150
1151         /*
1152          * A virtio driver normally uses at least 2 desc buffers
1153          * for Tx: the first for storing the header, and others
1154          * for storing the data.
1155          */
1156         if (unlikely(buf_len < dev->vhost_hlen)) {
1157                 buf_offset = dev->vhost_hlen - buf_len;
1158                 vec_idx++;
1159                 buf_addr = buf_vec[vec_idx].buf_addr;
1160                 buf_iova = buf_vec[vec_idx].buf_iova;
1161                 buf_len = buf_vec[vec_idx].buf_len;
1162                 buf_avail  = buf_len - buf_offset;
1163         } else if (buf_len == dev->vhost_hlen) {
1164                 if (unlikely(++vec_idx >= nr_vec))
1165                         goto out;
1166                 buf_addr = buf_vec[vec_idx].buf_addr;
1167                 buf_iova = buf_vec[vec_idx].buf_iova;
1168                 buf_len = buf_vec[vec_idx].buf_len;
1169
1170                 buf_offset = 0;
1171                 buf_avail = buf_len;
1172         } else {
1173                 buf_offset = dev->vhost_hlen;
1174                 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;
1175         }
1176
1177         rte_prefetch0((void *)(uintptr_t)
1178                         (buf_addr + buf_offset));
1179
1180         PRINT_PACKET(dev,
1181                         (uintptr_t)(buf_addr + buf_offset),
1182                         (uint32_t)buf_avail, 0);
1183
1184         mbuf_offset = 0;
1185         mbuf_avail  = m->buf_len - RTE_PKTMBUF_HEADROOM;
1186         while (1) {
1187                 uint64_t hpa;
1188
1189                 cpy_len = RTE_MIN(buf_avail, mbuf_avail);
1190
1191                 /*
1192                  * A desc buf might across two host physical pages that are
1193                  * not continuous. In such case (gpa_to_hpa returns 0), data
1194                  * will be copied even though zero copy is enabled.
1195                  */
1196                 if (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,
1197                                         buf_iova + buf_offset, cpy_len)))) {
1198                         cur->data_len = cpy_len;
1199                         cur->data_off = 0;
1200                         cur->buf_addr =
1201                                 (void *)(uintptr_t)(buf_addr + buf_offset);
1202                         cur->buf_iova = hpa;
1203
1204                         /*
1205                          * In zero copy mode, one mbuf can only reference data
1206                          * for one or partial of one desc buff.
1207                          */
1208                         mbuf_avail = cpy_len;
1209                 } else {
1210                         if (likely(cpy_len > MAX_BATCH_LEN ||
1211                                    vq->batch_copy_nb_elems >= vq->size ||
1212                                    (hdr && cur == m))) {
1213                                 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
1214                                                                    mbuf_offset),
1215                                            (void *)((uintptr_t)(buf_addr +
1216                                                            buf_offset)),
1217                                            cpy_len);
1218                         } else {
1219                                 batch_copy[vq->batch_copy_nb_elems].dst =
1220                                         rte_pktmbuf_mtod_offset(cur, void *,
1221                                                                 mbuf_offset);
1222                                 batch_copy[vq->batch_copy_nb_elems].src =
1223                                         (void *)((uintptr_t)(buf_addr +
1224                                                                 buf_offset));
1225                                 batch_copy[vq->batch_copy_nb_elems].len =
1226                                         cpy_len;
1227                                 vq->batch_copy_nb_elems++;
1228                         }
1229                 }
1230
1231                 mbuf_avail  -= cpy_len;
1232                 mbuf_offset += cpy_len;
1233                 buf_avail -= cpy_len;
1234                 buf_offset += cpy_len;
1235
1236                 /* This buf reaches to its end, get the next one */
1237                 if (buf_avail == 0) {
1238                         if (++vec_idx >= nr_vec)
1239                                 break;
1240
1241                         buf_addr = buf_vec[vec_idx].buf_addr;
1242                         buf_iova = buf_vec[vec_idx].buf_iova;
1243                         buf_len = buf_vec[vec_idx].buf_len;
1244
1245                         /*
1246                          * Prefecth desc n + 1 buffer while
1247                          * desc n buffer is processed.
1248                          */
1249                         if (vec_idx + 1 < nr_vec)
1250                                 rte_prefetch0((void *)(uintptr_t)
1251                                                 buf_vec[vec_idx + 1].buf_addr);
1252
1253                         buf_offset = 0;
1254                         buf_avail  = buf_len;
1255
1256                         PRINT_PACKET(dev, (uintptr_t)buf_addr,
1257                                         (uint32_t)buf_avail, 0);
1258                 }
1259
1260                 /*
1261                  * This mbuf reaches to its end, get a new one
1262                  * to hold more data.
1263                  */
1264                 if (mbuf_avail == 0) {
1265                         cur = rte_pktmbuf_alloc(mbuf_pool);
1266                         if (unlikely(cur == NULL)) {
1267                                 RTE_LOG(ERR, VHOST_DATA, "Failed to "
1268                                         "allocate memory for mbuf.\n");
1269                                 error = -1;
1270                                 goto out;
1271                         }
1272                         if (unlikely(dev->dequeue_zero_copy))
1273                                 rte_mbuf_refcnt_update(cur, 1);
1274
1275                         prev->next = cur;
1276                         prev->data_len = mbuf_offset;
1277                         m->nb_segs += 1;
1278                         m->pkt_len += mbuf_offset;
1279                         prev = cur;
1280
1281                         mbuf_offset = 0;
1282                         mbuf_avail  = cur->buf_len - RTE_PKTMBUF_HEADROOM;
1283                 }
1284         }
1285
1286         prev->data_len = mbuf_offset;
1287         m->pkt_len    += mbuf_offset;
1288
1289         if (hdr)
1290                 vhost_dequeue_offload(hdr, m);
1291
1292 out:
1293
1294         return error;
1295 }
1296
1297 static __rte_always_inline struct zcopy_mbuf *
1298 get_zmbuf(struct vhost_virtqueue *vq)
1299 {
1300         uint16_t i;
1301         uint16_t last;
1302         int tries = 0;
1303
1304         /* search [last_zmbuf_idx, zmbuf_size) */
1305         i = vq->last_zmbuf_idx;
1306         last = vq->zmbuf_size;
1307
1308 again:
1309         for (; i < last; i++) {
1310                 if (vq->zmbufs[i].in_use == 0) {
1311                         vq->last_zmbuf_idx = i + 1;
1312                         vq->zmbufs[i].in_use = 1;
1313                         return &vq->zmbufs[i];
1314                 }
1315         }
1316
1317         tries++;
1318         if (tries == 1) {
1319                 /* search [0, last_zmbuf_idx) */
1320                 i = 0;
1321                 last = vq->last_zmbuf_idx;
1322                 goto again;
1323         }
1324
1325         return NULL;
1326 }
1327
1328 static __rte_always_inline uint16_t
1329 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
1330         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1331 {
1332         uint16_t i;
1333         uint16_t free_entries;
1334
1335         if (unlikely(dev->dequeue_zero_copy)) {
1336                 struct zcopy_mbuf *zmbuf, *next;
1337
1338                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1339                      zmbuf != NULL; zmbuf = next) {
1340                         next = TAILQ_NEXT(zmbuf, next);
1341
1342                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1343                                 update_shadow_used_ring_split(vq,
1344                                                 zmbuf->desc_idx, 0);
1345                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1346                                 restore_mbuf(zmbuf->mbuf);
1347                                 rte_pktmbuf_free(zmbuf->mbuf);
1348                                 put_zmbuf(zmbuf);
1349                                 vq->nr_zmbuf -= 1;
1350                         }
1351                 }
1352
1353                 if (likely(vq->shadow_used_idx)) {
1354                         flush_shadow_used_ring_split(dev, vq);
1355                         vhost_vring_call_split(dev, vq);
1356                 }
1357         }
1358
1359         rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
1360
1361         free_entries = *((volatile uint16_t *)&vq->avail->idx) -
1362                         vq->last_avail_idx;
1363         if (free_entries == 0)
1364                 return 0;
1365
1366         /*
1367          * The ordering between avail index and
1368          * desc reads needs to be enforced.
1369          */
1370         rte_smp_rmb();
1371
1372         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1373
1374         count = RTE_MIN(count, MAX_PKT_BURST);
1375         count = RTE_MIN(count, free_entries);
1376         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1377                         dev->vid, count);
1378
1379         for (i = 0; i < count; i++) {
1380                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1381                 uint16_t head_idx;
1382                 uint32_t dummy_len;
1383                 uint16_t nr_vec = 0;
1384                 int err;
1385
1386                 if (unlikely(fill_vec_buf_split(dev, vq,
1387                                                 vq->last_avail_idx + i,
1388                                                 &nr_vec, buf_vec,
1389                                                 &head_idx, &dummy_len,
1390                                                 VHOST_ACCESS_RO) < 0))
1391                         break;
1392
1393                 if (likely(dev->dequeue_zero_copy == 0))
1394                         update_shadow_used_ring_split(vq, head_idx, 0);
1395
1396                 rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
1397
1398                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
1399                 if (unlikely(pkts[i] == NULL)) {
1400                         RTE_LOG(ERR, VHOST_DATA,
1401                                 "Failed to allocate memory for mbuf.\n");
1402                         break;
1403                 }
1404
1405                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1406                                 mbuf_pool);
1407                 if (unlikely(err)) {
1408                         rte_pktmbuf_free(pkts[i]);
1409                         break;
1410                 }
1411
1412                 if (unlikely(dev->dequeue_zero_copy)) {
1413                         struct zcopy_mbuf *zmbuf;
1414
1415                         zmbuf = get_zmbuf(vq);
1416                         if (!zmbuf) {
1417                                 rte_pktmbuf_free(pkts[i]);
1418                                 break;
1419                         }
1420                         zmbuf->mbuf = pkts[i];
1421                         zmbuf->desc_idx = head_idx;
1422
1423                         /*
1424                          * Pin lock the mbuf; we will check later to see
1425                          * whether the mbuf is freed (when we are the last
1426                          * user) or not. If that's the case, we then could
1427                          * update the used ring safely.
1428                          */
1429                         rte_mbuf_refcnt_update(pkts[i], 1);
1430
1431                         vq->nr_zmbuf += 1;
1432                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1433                 }
1434         }
1435         vq->last_avail_idx += i;
1436
1437         if (likely(dev->dequeue_zero_copy == 0)) {
1438                 do_data_copy_dequeue(vq);
1439                 if (unlikely(i < count))
1440                         vq->shadow_used_idx = i;
1441                 if (likely(vq->shadow_used_idx)) {
1442                         flush_shadow_used_ring_split(dev, vq);
1443                         vhost_vring_call_split(dev, vq);
1444                 }
1445         }
1446
1447         return i;
1448 }
1449
1450 static __rte_always_inline uint16_t
1451 virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
1452         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1453 {
1454         uint16_t i;
1455
1456         rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
1457
1458         if (unlikely(dev->dequeue_zero_copy)) {
1459                 struct zcopy_mbuf *zmbuf, *next;
1460
1461                 for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
1462                      zmbuf != NULL; zmbuf = next) {
1463                         next = TAILQ_NEXT(zmbuf, next);
1464
1465                         if (mbuf_is_consumed(zmbuf->mbuf)) {
1466                                 update_shadow_used_ring_packed(vq,
1467                                                 zmbuf->desc_idx,
1468                                                 0,
1469                                                 zmbuf->desc_count);
1470
1471                                 TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
1472                                 restore_mbuf(zmbuf->mbuf);
1473                                 rte_pktmbuf_free(zmbuf->mbuf);
1474                                 put_zmbuf(zmbuf);
1475                                 vq->nr_zmbuf -= 1;
1476                         }
1477                 }
1478
1479                 if (likely(vq->shadow_used_idx)) {
1480                         flush_shadow_used_ring_packed(dev, vq);
1481                         vhost_vring_call_packed(dev, vq);
1482                 }
1483         }
1484
1485         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
1486
1487         count = RTE_MIN(count, MAX_PKT_BURST);
1488         VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
1489                         dev->vid, count);
1490
1491         for (i = 0; i < count; i++) {
1492                 struct buf_vector buf_vec[BUF_VECTOR_MAX];
1493                 uint16_t buf_id;
1494                 uint32_t dummy_len;
1495                 uint16_t desc_count, nr_vec = 0;
1496                 int err;
1497
1498                 if (unlikely(fill_vec_buf_packed(dev, vq,
1499                                                 vq->last_avail_idx, &desc_count,
1500                                                 buf_vec, &nr_vec,
1501                                                 &buf_id, &dummy_len,
1502                                                 VHOST_ACCESS_RO) < 0))
1503                         break;
1504
1505                 if (likely(dev->dequeue_zero_copy == 0))
1506                         update_shadow_used_ring_packed(vq, buf_id, 0,
1507                                         desc_count);
1508
1509                 rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr);
1510
1511                 pkts[i] = rte_pktmbuf_alloc(mbuf_pool);
1512                 if (unlikely(pkts[i] == NULL)) {
1513                         RTE_LOG(ERR, VHOST_DATA,
1514                                 "Failed to allocate memory for mbuf.\n");
1515                         break;
1516                 }
1517
1518                 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],
1519                                 mbuf_pool);
1520                 if (unlikely(err)) {
1521                         rte_pktmbuf_free(pkts[i]);
1522                         break;
1523                 }
1524
1525                 if (unlikely(dev->dequeue_zero_copy)) {
1526                         struct zcopy_mbuf *zmbuf;
1527
1528                         zmbuf = get_zmbuf(vq);
1529                         if (!zmbuf) {
1530                                 rte_pktmbuf_free(pkts[i]);
1531                                 break;
1532                         }
1533                         zmbuf->mbuf = pkts[i];
1534                         zmbuf->desc_idx = buf_id;
1535                         zmbuf->desc_count = desc_count;
1536
1537                         /*
1538                          * Pin lock the mbuf; we will check later to see
1539                          * whether the mbuf is freed (when we are the last
1540                          * user) or not. If that's the case, we then could
1541                          * update the used ring safely.
1542                          */
1543                         rte_mbuf_refcnt_update(pkts[i], 1);
1544
1545                         vq->nr_zmbuf += 1;
1546                         TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);
1547                 }
1548
1549                 vq->last_avail_idx += desc_count;
1550                 if (vq->last_avail_idx >= vq->size) {
1551                         vq->last_avail_idx -= vq->size;
1552                         vq->avail_wrap_counter ^= 1;
1553                 }
1554         }
1555
1556         if (likely(dev->dequeue_zero_copy == 0)) {
1557                 do_data_copy_dequeue(vq);
1558                 if (unlikely(i < count))
1559                         vq->shadow_used_idx = i;
1560                 if (likely(vq->shadow_used_idx)) {
1561                         flush_shadow_used_ring_packed(dev, vq);
1562                         vhost_vring_call_packed(dev, vq);
1563                 }
1564         }
1565
1566         return i;
1567 }
1568
1569 uint16_t
1570 rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
1571         struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
1572 {
1573         struct virtio_net *dev;
1574         struct rte_mbuf *rarp_mbuf = NULL;
1575         struct vhost_virtqueue *vq;
1576
1577         dev = get_device(vid);
1578         if (!dev)
1579                 return 0;
1580
1581         if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
1582                 RTE_LOG(ERR, VHOST_DATA,
1583                         "(%d) %s: built-in vhost net backend is disabled.\n",
1584                         dev->vid, __func__);
1585                 return 0;
1586         }
1587
1588         if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
1589                 RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
1590                         dev->vid, __func__, queue_id);
1591                 return 0;
1592         }
1593
1594         vq = dev->virtqueue[queue_id];
1595
1596         if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
1597                 return 0;
1598
1599         if (unlikely(vq->enabled == 0)) {
1600                 count = 0;
1601                 goto out_access_unlock;
1602         }
1603
1604         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1605                 vhost_user_iotlb_rd_lock(vq);
1606
1607         if (unlikely(vq->access_ok == 0))
1608                 if (unlikely(vring_translate(dev, vq) < 0)) {
1609                         count = 0;
1610                         goto out;
1611                 }
1612
1613         /*
1614          * Construct a RARP broadcast packet, and inject it to the "pkts"
1615          * array, to looks like that guest actually send such packet.
1616          *
1617          * Check user_send_rarp() for more information.
1618          *
1619          * broadcast_rarp shares a cacheline in the virtio_net structure
1620          * with some fields that are accessed during enqueue and
1621          * rte_atomic16_cmpset() causes a write if using cmpxchg. This could
1622          * result in false sharing between enqueue and dequeue.
1623          *
1624          * Prevent unnecessary false sharing by reading broadcast_rarp first
1625          * and only performing cmpset if the read indicates it is likely to
1626          * be set.
1627          */
1628         if (unlikely(rte_atomic16_read(&dev->broadcast_rarp) &&
1629                         rte_atomic16_cmpset((volatile uint16_t *)
1630                                 &dev->broadcast_rarp.cnt, 1, 0))) {
1631
1632                 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
1633                 if (rarp_mbuf == NULL) {
1634                         RTE_LOG(ERR, VHOST_DATA,
1635                                 "Failed to make RARP packet.\n");
1636                         count = 0;
1637                         goto out;
1638                 }
1639                 count -= 1;
1640         }
1641
1642         if (vq_is_packed(dev))
1643                 count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);
1644         else
1645                 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);
1646
1647 out:
1648         if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
1649                 vhost_user_iotlb_rd_unlock(vq);
1650
1651 out_access_unlock:
1652         rte_spinlock_unlock(&vq->access_lock);
1653
1654         if (unlikely(rarp_mbuf != NULL)) {
1655                 /*
1656                  * Inject it to the head of "pkts" array, so that switch's mac
1657                  * learning table will get updated first.
1658                  */
1659                 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *));
1660                 pkts[0] = rarp_mbuf;
1661                 count += 1;
1662         }
1663
1664         return count;
1665 }