2 *------------------------------------------------------------------
5 * Copyright (c) 2014 Cisco and/or its affiliates.
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at:
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *------------------------------------------------------------------
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
36 #include <vnet/ip/ip.h>
38 #include <vnet/ethernet/ethernet.h>
40 #include <vnet/devices/virtio/vhost-user.h>
42 #define VHOST_USER_DEBUG_SOCKET 0
43 #define VHOST_USER_DEBUG_VQ 0
45 /* Set to get virtio_net_hdr in buffer pre-data
46 details will be shown in packet trace */
47 #define VHOST_USER_COPY_TX_HDR 0
49 #if VHOST_USER_DEBUG_SOCKET == 1
50 #define DBG_SOCK(args...) clib_warning(args);
52 #define DBG_SOCK(args...)
55 #if VHOST_USER_DEBUG_VQ == 1
56 #define DBG_VQ(args...) clib_warning(args);
58 #define DBG_VQ(args...)
61 vlib_node_registration_t vhost_user_input_node;
63 #define foreach_vhost_user_tx_func_error \
65 _(NOT_READY, "vhost user state error") \
66 _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
67 _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \
68 _(MMAP_FAIL, "mmap failure") \
69 _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")
73 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f,
74 foreach_vhost_user_tx_func_error
76 VHOST_USER_TX_FUNC_N_ERROR,
77 } vhost_user_tx_func_error_t;
79 static char *vhost_user_tx_func_error_strings[] = {
81 foreach_vhost_user_tx_func_error
85 #define foreach_vhost_user_input_func_error \
86 _(NO_ERROR, "no error") \
87 _(NO_BUFFER, "no available buffer") \
88 _(MMAP_FAIL, "mmap failure") \
89 _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
90 _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
91 _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
95 #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
96 foreach_vhost_user_input_func_error
98 VHOST_USER_INPUT_FUNC_N_ERROR,
99 } vhost_user_input_func_error_t;
101 static char *vhost_user_input_func_error_strings[] = {
103 foreach_vhost_user_input_func_error
108 static vhost_user_main_t vhost_user_main = {
112 VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = {
113 .name = "vhost-user",
118 format_vhost_user_interface_name (u8 * s, va_list * args)
120 u32 i = va_arg (*args, u32);
121 u32 show_dev_instance = ~0;
122 vhost_user_main_t *vum = &vhost_user_main;
124 if (i < vec_len (vum->show_dev_instance_by_real_dev_instance))
125 show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i];
127 if (show_dev_instance != ~0)
128 i = show_dev_instance;
130 s = format (s, "VirtualEthernet0/0/%d", i);
135 vhost_user_name_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance)
137 vhost_user_main_t *vum = &vhost_user_main;
139 vec_validate_init_empty (vum->show_dev_instance_by_real_dev_instance,
140 hi->dev_instance, ~0);
142 vum->show_dev_instance_by_real_dev_instance[hi->dev_instance] =
145 DBG_SOCK ("renumbered vhost-user interface dev_instance %d to %d",
146 hi->dev_instance, new_dev_instance);
153 map_guest_mem (vhost_user_intf_t * vui, uword addr)
156 for (i = 0; i < vui->nregions; i++)
158 if ((vui->regions[i].guest_phys_addr <= addr) &&
159 ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
162 return (void *) (vui->region_mmap_addr[i] + addr -
163 vui->regions[i].guest_phys_addr);
166 DBG_VQ ("failed to map guest mem addr %llx", addr);
171 map_user_mem (vhost_user_intf_t * vui, uword addr)
174 for (i = 0; i < vui->nregions; i++)
176 if ((vui->regions[i].userspace_addr <= addr) &&
177 ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) >
180 return (void *) (vui->region_mmap_addr[i] + addr -
181 vui->regions[i].userspace_addr);
188 get_huge_page_size (int fd)
196 unmap_all_mem_regions (vhost_user_intf_t * vui)
199 for (i = 0; i < vui->nregions; i++)
201 if (vui->region_mmap_addr[i] != (void *) -1)
204 long page_sz = get_huge_page_size (vui->region_mmap_fd[i]);
206 ssize_t map_sz = (vui->regions[i].memory_size +
207 vui->regions[i].mmap_offset +
208 page_sz) & ~(page_sz - 1);
211 munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset,
215 ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i,
216 vui->region_mmap_addr[i], map_sz, page_sz);
218 vui->region_mmap_addr[i] = (void *) -1;
222 clib_warning ("failed to unmap memory region (errno %d)",
225 close (vui->region_mmap_fd[i]);
232 static clib_error_t *
233 vhost_user_callfd_read_ready (unix_file_t * uf)
235 __attribute__ ((unused)) int n;
237 n = read (uf->file_descriptor, ((char *) &buff), 8);
242 vhost_user_if_disconnect (vhost_user_intf_t * vui)
244 vhost_user_main_t *vum = &vhost_user_main;
245 vnet_main_t *vnm = vnet_get_main ();
248 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
250 if (vui->unix_file_index != ~0)
252 unix_file_del (&unix_main, unix_main.file_pool + vui->unix_file_index);
253 vui->unix_file_index = ~0;
256 close (vui->unix_fd);
258 hash_unset (vum->vhost_user_interface_index_by_sock_fd, vui->unix_fd);
259 hash_unset (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd);
262 for (q = 0; q < vui->num_vrings; q++)
264 vui->vrings[q].desc = NULL;
265 vui->vrings[q].avail = NULL;
266 vui->vrings[q].used = NULL;
267 vui->vrings[q].log_guest_addr = 0;
268 vui->vrings[q].log_used = 0;
271 unmap_all_mem_regions (vui);
272 DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index);
275 #define VHOST_LOG_PAGE 0x1000
277 vhost_user_log_dirty_pages (vhost_user_intf_t * vui, u64 addr, u64 len)
279 if (PREDICT_TRUE (vui->log_base_addr == 0
280 || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL))))
284 if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size))
286 DBG_SOCK ("vhost_user_log_dirty_pages(): out of range\n");
290 CLIB_MEMORY_BARRIER ();
291 u64 page = addr / VHOST_LOG_PAGE;
292 while (page * VHOST_LOG_PAGE < addr + len)
294 ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8;
299 #define vhost_user_log_dirty_ring(vui, vq, member) \
300 if (PREDICT_FALSE(vq->log_used)) { \
301 vhost_user_log_dirty_pages(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \
302 sizeof(vq->used->member)); \
305 static clib_error_t *
306 vhost_user_socket_read (unix_file_t * uf)
309 int fd, number_of_fds = 0;
310 int fds[VHOST_MEMORY_MAX_NREGIONS];
311 vhost_user_msg_t msg;
314 vhost_user_main_t *vum = &vhost_user_main;
315 vhost_user_intf_t *vui;
316 struct cmsghdr *cmsg;
319 unix_file_t template = { 0 };
320 vnet_main_t *vnm = vnet_get_main ();
322 p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
323 uf->file_descriptor);
326 DBG_SOCK ("FD %d doesn't belong to any interface", uf->file_descriptor);
330 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
332 char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))];
334 memset (&mh, 0, sizeof (mh));
335 memset (control, 0, sizeof (control));
337 for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++)
340 /* set the payload */
341 iov[0].iov_base = (void *) &msg;
342 iov[0].iov_len = VHOST_USER_MSG_HDR_SZ;
346 mh.msg_control = control;
347 mh.msg_controllen = sizeof (control);
349 n = recvmsg (uf->file_descriptor, &mh, 0);
351 if (n != VHOST_USER_MSG_HDR_SZ)
354 if (mh.msg_flags & MSG_CTRUNC)
359 cmsg = CMSG_FIRSTHDR (&mh);
361 if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
362 (cmsg->cmsg_type == SCM_RIGHTS) &&
363 (cmsg->cmsg_len - CMSG_LEN (0) <=
364 VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
366 number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
367 clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
370 /* version 1, no reply bit set */
371 if ((msg.flags & 7) != 1)
373 DBG_SOCK ("malformed message received. closing socket");
378 int rv __attribute__ ((unused));
379 /* $$$$ pay attention to rv */
380 rv = read (uf->file_descriptor, ((char *) &msg) + n, msg.size);
385 case VHOST_USER_GET_FEATURES:
386 DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES", vui->hw_if_index);
389 msg.u64 = (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
390 (1 << FEAT_VIRTIO_F_ANY_LAYOUT) |
391 (1 << FEAT_VIRTIO_F_INDIRECT_DESC) |
392 (1 << FEAT_VHOST_F_LOG_ALL) |
393 (1 << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
394 (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
395 (1UL << FEAT_VIRTIO_F_VERSION_1);
396 msg.u64 &= vui->feature_mask;
398 msg.size = sizeof (msg.u64);
401 case VHOST_USER_SET_FEATURES:
402 DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx",
403 vui->hw_if_index, msg.u64);
405 vui->features = msg.u64;
407 if (vui->features & (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF))
408 vui->virtio_net_hdr_sz = 12;
410 vui->virtio_net_hdr_sz = 10;
413 (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0;
415 ASSERT (vui->virtio_net_hdr_sz < VLIB_BUFFER_PRE_DATA_SIZE);
416 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
419 for (q = 0; q < 2; q++)
421 vui->vrings[q].desc = 0;
422 vui->vrings[q].avail = 0;
423 vui->vrings[q].used = 0;
424 vui->vrings[q].log_guest_addr = 0;
425 vui->vrings[q].log_used = 0;
428 DBG_SOCK ("interface %d disconnected", vui->sw_if_index);
432 case VHOST_USER_SET_MEM_TABLE:
433 DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
434 vui->hw_if_index, msg.memory.nregions);
436 if ((msg.memory.nregions < 1) ||
437 (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
440 DBG_SOCK ("number of mem regions must be between 1 and %i",
441 VHOST_MEMORY_MAX_NREGIONS);
446 if (msg.memory.nregions != number_of_fds)
448 DBG_SOCK ("each memory region must have FD");
451 unmap_all_mem_regions (vui);
452 for (i = 0; i < msg.memory.nregions; i++)
454 clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
455 sizeof (vhost_user_memory_region_t));
457 long page_sz = get_huge_page_size (fds[i]);
459 /* align size to 2M page */
460 ssize_t map_sz = (vui->regions[i].memory_size +
461 vui->regions[i].mmap_offset +
462 page_sz) & ~(page_sz - 1);
464 vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
465 MAP_SHARED, fds[i], 0);
468 ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
469 "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i],
472 if (vui->region_mmap_addr[i] == MAP_FAILED)
474 clib_warning ("failed to map memory. errno is %d", errno);
477 vui->region_mmap_addr[i] += vui->regions[i].mmap_offset;
478 vui->region_mmap_fd[i] = fds[i];
480 vui->nregions = msg.memory.nregions;
483 case VHOST_USER_SET_VRING_NUM:
484 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
485 vui->hw_if_index, msg.state.index, msg.state.num);
487 if ((msg.state.num > 32768) || /* maximum ring size is 32768 */
488 (msg.state.num == 0) || /* it cannot be zero */
489 (msg.state.num % 2)) /* must be power of 2 */
491 vui->vrings[msg.state.index].qsz = msg.state.num;
494 case VHOST_USER_SET_VRING_ADDR:
495 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
496 vui->hw_if_index, msg.state.index);
498 vui->vrings[msg.state.index].desc = (vring_desc_t *)
499 map_user_mem (vui, msg.addr.desc_user_addr);
500 vui->vrings[msg.state.index].used = (vring_used_t *)
501 map_user_mem (vui, msg.addr.used_user_addr);
502 vui->vrings[msg.state.index].avail = (vring_avail_t *)
503 map_user_mem (vui, msg.addr.avail_user_addr);
505 if ((vui->vrings[msg.state.index].desc == NULL) ||
506 (vui->vrings[msg.state.index].used == NULL) ||
507 (vui->vrings[msg.state.index].avail == NULL))
509 DBG_SOCK ("failed to map user memory for hw_if_index %d",
514 vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
515 vui->vrings[msg.state.index].log_used =
516 (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;
518 /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
519 the ring is initialized in an enabled state. */
521 if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES)))
523 vui->vrings[msg.state.index].enabled = 1;
526 vui->vrings[msg.state.index].last_used_idx =
527 vui->vrings[msg.state.index].used->idx;
529 /* tell driver that we don't want interrupts */
530 vui->vrings[msg.state.index].used->flags |= 1;
533 case VHOST_USER_SET_OWNER:
534 DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index);
537 case VHOST_USER_RESET_OWNER:
538 DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index);
541 case VHOST_USER_SET_VRING_CALL:
542 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL u64 %d",
543 vui->hw_if_index, msg.u64);
545 q = (u8) (msg.u64 & 0xFF);
547 if (!(msg.u64 & 0x100))
549 if (number_of_fds != 1)
552 /* if there is old fd, delete it */
553 if (vui->vrings[q].callfd)
555 unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
556 vui->vrings[q].callfd_idx);
557 unix_file_del (&unix_main, uf);
559 vui->vrings[q].callfd = fds[0];
560 template.read_function = vhost_user_callfd_read_ready;
561 template.file_descriptor = fds[0];
562 vui->vrings[q].callfd_idx = unix_file_add (&unix_main, &template);
565 vui->vrings[q].callfd = -1;
568 case VHOST_USER_SET_VRING_KICK:
569 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK u64 %d",
570 vui->hw_if_index, msg.u64);
572 q = (u8) (msg.u64 & 0xFF);
574 if (!(msg.u64 & 0x100))
576 if (number_of_fds != 1)
579 vui->vrings[q].kickfd = fds[0];
582 vui->vrings[q].kickfd = -1;
585 case VHOST_USER_SET_VRING_ERR:
586 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR u64 %d",
587 vui->hw_if_index, msg.u64);
589 q = (u8) (msg.u64 & 0xFF);
591 if (!(msg.u64 & 0x100))
593 if (number_of_fds != 1)
601 vui->vrings[q].errfd = fd;
604 case VHOST_USER_SET_VRING_BASE:
605 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d",
606 vui->hw_if_index, msg.state.index, msg.state.num);
608 vui->vrings[msg.state.index].last_avail_idx = msg.state.num;
611 case VHOST_USER_GET_VRING_BASE:
612 DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
613 vui->hw_if_index, msg.state.index, msg.state.num);
615 /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
616 vui->vrings[msg.state.index].enabled = 0;
618 msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
620 msg.size = sizeof (msg.state);
623 case VHOST_USER_NONE:
624 DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index);
628 case VHOST_USER_SET_LOG_BASE:
630 DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index);
632 if (msg.size != sizeof (msg.log))
635 ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
636 msg.size, sizeof (msg.log));
641 (vui->protocol_features & (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD)))
644 ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
649 /* align size to 2M page */
650 long page_sz = get_huge_page_size (fd);
652 (msg.log.size + msg.log.offset + page_sz) & ~(page_sz - 1);
654 vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
658 ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
659 map_sz, msg.log.offset, fd, vui->log_base_addr);
661 if (vui->log_base_addr == MAP_FAILED)
663 clib_warning ("failed to map memory. errno is %d", errno);
667 vui->log_base_addr += msg.log.offset;
668 vui->log_size = msg.log.size;
671 msg.size = sizeof (msg.u64);
676 case VHOST_USER_SET_LOG_FD:
677 DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index);
681 case VHOST_USER_GET_PROTOCOL_FEATURES:
682 DBG_SOCK ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES",
686 msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
687 msg.size = sizeof (msg.u64);
690 case VHOST_USER_SET_PROTOCOL_FEATURES:
691 DBG_SOCK ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%lx",
692 vui->hw_if_index, msg.u64);
694 vui->protocol_features = msg.u64;
698 case VHOST_USER_SET_VRING_ENABLE:
699 DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE, enable: %d",
700 vui->hw_if_index, msg.state.num);
701 vui->vrings[msg.state.index].enabled = msg.state.num;
705 DBG_SOCK ("unknown vhost-user message %d received. closing socket",
710 /* if we have pointers to descriptor table, go up */
712 vui->vrings[VHOST_NET_VRING_IDX_TX].desc &&
713 vui->vrings[VHOST_NET_VRING_IDX_RX].desc)
716 DBG_SOCK ("interface %d connected", vui->sw_if_index);
718 vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
719 VNET_HW_INTERFACE_FLAG_LINK_UP);
724 /* if we need to reply */
728 send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0);
729 if (n != (msg.size + VHOST_USER_MSG_HDR_SZ))
736 vhost_user_if_disconnect (vui);
740 static clib_error_t *
741 vhost_user_socket_error (unix_file_t * uf)
743 vhost_user_main_t *vum = &vhost_user_main;
744 vhost_user_intf_t *vui;
747 p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
748 uf->file_descriptor);
751 DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
755 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
757 vhost_user_if_disconnect (vui);
761 static clib_error_t *
762 vhost_user_socksvr_accept_ready (unix_file_t * uf)
764 int client_fd, client_len;
765 struct sockaddr_un client;
766 unix_file_t template = { 0 };
767 vhost_user_main_t *vum = &vhost_user_main;
768 vhost_user_intf_t *vui;
771 p = hash_get (vum->vhost_user_interface_index_by_listener_fd,
772 uf->file_descriptor);
775 DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
779 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
781 client_len = sizeof (client);
782 client_fd = accept (uf->file_descriptor,
783 (struct sockaddr *) &client,
784 (socklen_t *) & client_len);
787 return clib_error_return_unix (0, "accept");
789 template.read_function = vhost_user_socket_read;
790 template.error_function = vhost_user_socket_error;
791 template.file_descriptor = client_fd;
792 vui->unix_file_index = unix_file_add (&unix_main, &template);
794 vui->client_fd = client_fd;
795 hash_set (vum->vhost_user_interface_index_by_sock_fd, vui->client_fd,
796 vui - vum->vhost_user_interfaces);
801 static clib_error_t *
802 vhost_user_init (vlib_main_t * vm)
805 vhost_user_main_t *vum = &vhost_user_main;
806 vlib_thread_main_t *tm = vlib_get_thread_main ();
808 error = vlib_call_init_function (vm, ip4_init);
812 vum->vhost_user_interface_index_by_listener_fd =
813 hash_create (0, sizeof (uword));
814 vum->vhost_user_interface_index_by_sock_fd =
815 hash_create (0, sizeof (uword));
816 vum->vhost_user_interface_index_by_sw_if_index =
817 hash_create (0, sizeof (uword));
818 vum->coalesce_frames = 32;
819 vum->coalesce_time = 1e-3;
821 vec_validate_aligned (vum->rx_buffers, tm->n_vlib_mains - 1,
822 CLIB_CACHE_LINE_BYTES);
827 VLIB_INIT_FUNCTION (vhost_user_init);
829 static clib_error_t *
830 vhost_user_exit (vlib_main_t * vm)
836 VLIB_MAIN_LOOP_EXIT_FUNCTION (vhost_user_exit);
840 VHOST_USER_RX_NEXT_ETHERNET_INPUT,
841 VHOST_USER_RX_NEXT_DROP,
842 VHOST_USER_RX_N_NEXT,
850 #if VHOST_USER_COPY_TX_HDR == 1
851 virtio_net_hdr_t hdr;
853 } vhost_user_input_trace_t;
856 format_vhost_user_input_trace (u8 * s, va_list * va)
858 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
859 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
860 CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
861 vhost_user_main_t *vum = &vhost_user_main;
862 vhost_user_input_trace_t *t = va_arg (*va, vhost_user_input_trace_t *);
863 vhost_user_intf_t *vui = vec_elt_at_index (vum->vhost_user_interfaces,
866 vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, vui->sw_if_index);
868 #if VHOST_USER_COPY_TX_HDR == 1
869 uword indent = format_get_indent (s);
872 s = format (s, "%U virtqueue %d",
873 format_vnet_sw_interface_name, vnm, sw, t->virtqueue);
875 #if VHOST_USER_COPY_TX_HDR == 1
876 s = format (s, "\n%Uvirtio_net_hdr flags 0x%02x gso_type %u hdr_len %u",
877 format_white_space, indent,
878 t->hdr.flags, t->hdr.gso_type, t->hdr.hdr_len);
885 vhost_user_rx_trace (vlib_main_t * vm,
886 vlib_node_runtime_t * node,
887 vhost_user_intf_t * vui, i16 virtqueue)
890 vhost_user_main_t *vum = &vhost_user_main;
892 u32 next_index = VHOST_USER_RX_NEXT_ETHERNET_INPUT;
894 n_left = vec_len (vui->d_trace_buffers);
895 b = vui->d_trace_buffers;
901 vhost_user_input_trace_t *t0;
906 b0 = vlib_get_buffer (vm, bi0);
907 vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 0);
908 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
909 t0->virtqueue = virtqueue;
910 t0->device_index = vui - vum->vhost_user_interfaces;
911 #if VHOST_USER_COPY_TX_HDR == 1
912 clib_memcpy (&t0->hdr, b0->pre_data, sizeof (virtio_net_hdr_t));
920 vhost_user_send_call (vlib_main_t * vm, vhost_user_vring_t * vq)
922 vhost_user_main_t *vum = &vhost_user_main;
924 int rv __attribute__ ((unused));
925 /* $$$$ pay attention to rv */
926 rv = write (vq->callfd, &x, sizeof (x));
927 vq->n_since_last_int = 0;
928 vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time;
933 vhost_user_if_input (vlib_main_t * vm,
934 vhost_user_main_t * vum,
935 vhost_user_intf_t * vui, vlib_node_runtime_t * node)
937 vhost_user_vring_t *txvq = &vui->vrings[VHOST_NET_VRING_IDX_TX];
938 vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
939 uword n_rx_packets = 0, n_rx_bytes = 0;
941 u32 n_left_to_next, *to_next;
944 uword n_trace = vlib_get_trace_count (vm, node);
946 u32 cpu_index, rx_len, drops, flush;
947 f64 now = vlib_time_now (vm);
949 vec_reset_length (vui->d_trace_buffers);
951 /* no descriptor ptr - bail out */
952 if (PREDICT_FALSE (!txvq->desc || !txvq->avail || !txvq->enabled))
955 /* do we have pending intterupts ? */
956 if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
957 vhost_user_send_call (vm, txvq);
959 if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
960 vhost_user_send_call (vm, rxvq);
962 /* only bit 0 of avail.flags is used so we don't want to deal with this
963 interface if any other bit is set */
964 if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
967 n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
970 if (PREDICT_FALSE (n_left == 0))
973 if (PREDICT_FALSE (n_left == txvq->qsz))
975 //Informational error logging when VPP is not receiving packets fast enough
976 vlib_error_count (vm, node->node_index,
977 VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
980 if (PREDICT_FALSE (!vui->admin_up))
982 /* if intf is admin down, just drop all packets waiting in the ring */
983 txvq->last_avail_idx = txvq->last_used_idx = txvq->avail->idx;
984 CLIB_MEMORY_BARRIER ();
985 txvq->used->idx = txvq->last_used_idx;
986 vhost_user_log_dirty_ring (vui, txvq, idx);
987 vhost_user_send_call (vm, txvq);
991 qsz_mask = txvq->qsz - 1;
992 cpu_index = os_get_cpu_number ();
996 if (n_left > VLIB_FRAME_SIZE)
997 n_left = VLIB_FRAME_SIZE;
999 /* Allocate some buffers.
1000 * Note that buffers that are chained for jumbo
1001 * frames are allocated separately using a slower path.
1002 * The idea is to be certain to have enough buffers at least
1003 * to cycle through the descriptors without having to check for errors.
1004 * For jumbo frames, the bottleneck is memory copy anyway.
1006 if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
1008 vec_alloc (vum->rx_buffers[cpu_index], 2 * VLIB_FRAME_SIZE);
1010 if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
1011 flush = n_left; //Drop all input
1014 if (PREDICT_FALSE (_vec_len (vum->rx_buffers[cpu_index]) < n_left))
1016 u32 curr_len = _vec_len (vum->rx_buffers[cpu_index]);
1017 _vec_len (vum->rx_buffers[cpu_index]) +=
1018 vlib_buffer_alloc_from_free_list (vm,
1019 vum->rx_buffers[cpu_index] +
1021 2 * VLIB_FRAME_SIZE - curr_len,
1022 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1024 if (PREDICT_FALSE (n_left > _vec_len (vum->rx_buffers[cpu_index])))
1025 flush = n_left - _vec_len (vum->rx_buffers[cpu_index]);
1028 if (PREDICT_FALSE (flush))
1030 //Remove some input buffers
1033 vlib_error_count (vm, vhost_user_input_node.index,
1034 VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
1037 u16 desc_chain_head =
1038 txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
1039 txvq->last_avail_idx++;
1040 txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
1042 txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
1043 vhost_user_log_dirty_ring (vui, txvq,
1044 ring[txvq->last_used_idx & qsz_mask]);
1045 txvq->last_used_idx++;
1050 rx_len = vec_len (vum->rx_buffers[cpu_index]); //vector might be null
1053 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1055 while (n_left > 0 && n_left_to_next > 0)
1057 vlib_buffer_t *b_head, *b_current;
1058 u32 bi_head, bi_current;
1059 u16 desc_chain_head, desc_current;
1060 u8 error = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
1062 if (PREDICT_TRUE (n_left > 1))
1065 txvq->avail->ring[(txvq->last_avail_idx + 1) & qsz_mask];
1067 map_guest_mem (vui, txvq->desc[next_desc].addr);
1068 if (PREDICT_TRUE (buffer_addr != 0))
1069 CLIB_PREFETCH (buffer_addr, 64, STORE);
1071 u32 bi = vum->rx_buffers[cpu_index][rx_len - 2];
1072 vlib_prefetch_buffer_with_index (vm, bi, STORE);
1073 CLIB_PREFETCH (vlib_get_buffer (vm, bi)->data, 128, STORE);
1076 desc_chain_head = desc_current =
1077 txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
1078 bi_head = bi_current = vum->rx_buffers[cpu_index][--rx_len];
1079 b_head = b_current = vlib_get_buffer (vm, bi_head);
1080 vlib_buffer_chain_init (b_head);
1083 if (PREDICT_TRUE (vui->is_any_layout) ||
1084 (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
1085 !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)))
1087 /* ANYLAYOUT or single buffer */
1088 offset = vui->virtio_net_hdr_sz;
1092 /* CSR case without ANYLAYOUT, skip 1st buffer */
1093 offset = txvq->desc[desc_current].len;
1096 vring_desc_t *desc_table = txvq->desc;
1097 u32 desc_index = desc_current;
1099 if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
1101 desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr);
1103 if (PREDICT_FALSE (desc_table == 0))
1105 error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
1113 map_guest_mem (vui, desc_table[desc_index].addr);
1114 if (PREDICT_FALSE (buffer_addr == 0))
1116 error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
1121 (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
1123 CLIB_PREFETCH (&desc_table[desc_table[desc_index].next],
1124 sizeof (vring_desc_t), STORE);
1127 #if VHOST_USER_COPY_TX_HDR == 1
1128 if (PREDICT_TRUE (offset))
1129 clib_memcpy (b->pre_data, buffer_addr, sizeof (virtio_net_hdr_t)); /* 12 byte hdr is not used on tx */
1132 if (desc_table[desc_index].len > offset)
1134 u16 len = desc_table[desc_index].len - offset;
1135 u16 copied = vlib_buffer_chain_append_data_with_alloc (vm,
1136 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX,
1145 error = VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER;
1151 /* if next flag is set, take next desc in the chain */
1152 if ((desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
1153 desc_index = desc_table[desc_index].next;
1159 /* consume the descriptor and return it as used */
1160 txvq->last_avail_idx++;
1161 txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
1163 txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
1164 vhost_user_log_dirty_ring (vui, txvq,
1165 ring[txvq->last_used_idx & qsz_mask]);
1166 txvq->last_used_idx++;
1168 //It is important to free RX as fast as possible such that the TX
1169 //process does not drop packets
1170 if ((txvq->last_used_idx & 0x3f) == 0) // Every 64 packets
1171 txvq->used->idx = txvq->last_used_idx;
1173 if (PREDICT_FALSE (b_head->current_length < 14 &&
1174 error == VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
1175 error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;
1177 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
1179 vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
1180 vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1181 b_head->error = node->errors[error];
1183 if (PREDICT_FALSE (n_trace > n_rx_packets))
1184 vec_add1 (vui->d_trace_buffers, bi_head);
1186 if (PREDICT_FALSE (error))
1189 next0 = VHOST_USER_RX_NEXT_DROP;
1194 b_head->current_length +
1195 b_head->total_length_not_including_first_buffer;
1197 next0 = VHOST_USER_RX_NEXT_ETHERNET_INPUT;
1200 to_next[0] = bi_head;
1203 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1204 to_next, n_left_to_next,
1209 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1213 if (PREDICT_TRUE (vum->rx_buffers[cpu_index] != 0))
1214 _vec_len (vum->rx_buffers[cpu_index]) = rx_len;
1216 /* give buffers back to driver */
1217 CLIB_MEMORY_BARRIER ();
1218 txvq->used->idx = txvq->last_used_idx;
1219 vhost_user_log_dirty_ring (vui, txvq, idx);
1221 if (PREDICT_FALSE (vec_len (vui->d_trace_buffers) > 0))
1223 vhost_user_rx_trace (vm, node, vui, VHOST_NET_VRING_IDX_TX);
1224 vlib_set_trace_count (vm, node,
1225 n_trace - vec_len (vui->d_trace_buffers));
1228 /* interrupt (call) handling */
1229 if ((txvq->callfd > 0) && !(txvq->avail->flags & 1))
1231 txvq->n_since_last_int += n_rx_packets;
1233 if (txvq->n_since_last_int > vum->coalesce_frames)
1234 vhost_user_send_call (vm, txvq);
1237 if (PREDICT_FALSE (drops))
1239 vlib_increment_simple_counter
1240 (vnet_main.interface_main.sw_if_counters
1241 + VNET_INTERFACE_COUNTER_DROP, os_get_cpu_number (),
1242 vui->sw_if_index, drops);
1245 /* increase rx counters */
1246 vlib_increment_combined_counter
1247 (vnet_main.interface_main.combined_sw_if_counters
1248 + VNET_INTERFACE_COUNTER_RX,
1249 os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
1251 return n_rx_packets;
1255 vhost_user_input (vlib_main_t * vm,
1256 vlib_node_runtime_t * node, vlib_frame_t * f)
1258 vhost_user_main_t *vum = &vhost_user_main;
1260 dpdk_main_t *dm = &dpdk_main;
1261 u32 cpu_index = os_get_cpu_number ();
1263 vhost_user_intf_t *vui;
1264 uword n_rx_packets = 0;
1267 for (i = 0; i < vec_len (vum->vhost_user_interfaces); i++)
1269 vui = vec_elt_at_index (vum->vhost_user_interfaces, i);
1273 if ((i % dm->input_cpu_count) ==
1274 (cpu_index - dm->input_cpu_first_index))
1276 n_rx_packets += vhost_user_if_input (vm, vum, vui, node);
1279 return n_rx_packets;
1283 VLIB_REGISTER_NODE (vhost_user_input_node) = {
1284 .function = vhost_user_input,
1285 .type = VLIB_NODE_TYPE_INPUT,
1286 .name = "vhost-user-input",
1288 /* Will be enabled if/when hardware is detected. */
1289 .state = VLIB_NODE_STATE_DISABLED,
1291 .format_buffer = format_ethernet_header_with_length,
1292 .format_trace = format_vhost_user_input_trace,
1294 .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
1295 .error_strings = vhost_user_input_func_error_strings,
1297 .n_next_nodes = VHOST_USER_RX_N_NEXT,
1299 [VHOST_USER_RX_NEXT_DROP] = "error-drop",
1300 [VHOST_USER_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
1304 VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input)
1308 vhost_user_intfc_tx (vlib_main_t * vm,
1309 vlib_node_runtime_t * node, vlib_frame_t * frame)
1311 u32 *buffers = vlib_frame_args (frame);
1313 vhost_user_main_t *vum = &vhost_user_main;
1314 uword n_packets = 0;
1315 vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
1316 vhost_user_intf_t *vui =
1317 vec_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
1318 vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
1320 u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;
1322 n_left = n_packets = frame->n_vectors;
1324 if (PREDICT_FALSE (!vui->is_up))
1328 (!rxvq->desc || !rxvq->avail || vui->sock_errno != 0 || !rxvq->enabled))
1330 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
1334 if (PREDICT_FALSE (vui->lockp != 0))
1336 while (__sync_lock_test_and_set (vui->lockp, 1))
1340 /* only bit 0 of avail.flags is used so we don't want to deal with this
1341 interface if any other bit is set */
1342 if (PREDICT_FALSE (rxvq->avail->flags & 0xFFFE))
1344 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
1348 if (PREDICT_FALSE ((rxvq->avail->idx == rxvq->last_avail_idx)))
1350 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1354 qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */
1358 vlib_buffer_t *b0, *current_b0;
1359 u16 desc_head, desc_index, desc_len;
1360 vring_desc_t *desc_table;
1364 b0 = vlib_get_buffer (vm, buffers[0]);
1367 if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
1369 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1373 desc_table = rxvq->desc;
1374 desc_head = desc_index =
1375 rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
1376 if (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)
1379 (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
1381 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
1386 map_guest_mem (vui, rxvq->desc[desc_index].addr))))
1388 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1394 desc_len = vui->virtio_net_hdr_sz;
1397 (!(buffer_addr = map_guest_mem (vui, desc_table[desc_index].addr))))
1399 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1402 buffer_len = desc_table[desc_index].len;
1404 CLIB_PREFETCH (buffer_addr,
1405 clib_min (buffer_len, 2 * CLIB_CACHE_LINE_BYTES), STORE);
1407 virtio_net_hdr_mrg_rxbuf_t *hdr =
1408 (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
1410 hdr->hdr.gso_type = 0;
1411 if (vui->virtio_net_hdr_sz == 12)
1412 hdr->num_buffers = 1;
1414 vhost_user_log_dirty_pages (vui, desc_table[desc_index].addr,
1415 vui->virtio_net_hdr_sz);
1417 u16 bytes_left = b0->current_length;
1418 buffer_addr += vui->virtio_net_hdr_sz;
1419 buffer_len -= vui->virtio_net_hdr_sz;
1425 if (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT)
1427 current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
1428 bytes_left = current_b0->current_length;
1437 if (buffer_len == 0)
1439 if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT)
1441 //Next one is chained
1442 desc_index = desc_table[desc_index].next;
1445 map_guest_mem (vui, desc_table[desc_index].addr))))
1447 rxvq->last_used_idx -= hdr->num_buffers - 1;
1448 rxvq->last_avail_idx -= hdr->num_buffers - 1;
1449 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1452 buffer_len = desc_table[desc_index].len;
1454 else if (vui->virtio_net_hdr_sz == 12) //MRG is available
1456 //Move from available to used buffer
1457 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id =
1459 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len =
1461 vhost_user_log_dirty_ring (vui, rxvq,
1462 ring[rxvq->last_used_idx &
1464 rxvq->last_avail_idx++;
1465 rxvq->last_used_idx++;
1469 (rxvq->last_avail_idx == rxvq->avail->idx))
1471 //Dequeue queued descriptors for this packet
1472 rxvq->last_used_idx -= hdr->num_buffers - 1;
1473 rxvq->last_avail_idx -= hdr->num_buffers - 1;
1474 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1478 desc_table = rxvq->desc;
1479 desc_head = desc_index =
1480 rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
1482 (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
1484 //It is seriously unlikely that a driver will put indirect descriptor
1485 //after non-indirect descriptor.
1487 (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
1489 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
1495 rxvq->desc[desc_index].addr))))
1497 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1505 map_guest_mem (vui, desc_table[desc_index].addr))))
1507 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1510 buffer_len = desc_table[desc_index].len;
1511 CLIB_PREFETCH (buffer_addr,
1512 clib_min (buffer_len,
1513 2 * CLIB_CACHE_LINE_BYTES), STORE);
1517 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
1522 u16 bytes_to_copy = bytes_left;
1524 (bytes_to_copy > buffer_len) ? buffer_len : bytes_to_copy;
1525 clib_memcpy (buffer_addr,
1526 vlib_buffer_get_current (current_b0) +
1527 current_b0->current_length - bytes_left,
1530 vhost_user_log_dirty_pages (vui,
1531 desc_table[desc_index].addr +
1532 desc_table[desc_index].len -
1533 bytes_left - bytes_to_copy,
1536 bytes_left -= bytes_to_copy;
1537 buffer_len -= bytes_to_copy;
1538 buffer_addr += bytes_to_copy;
1539 desc_len += bytes_to_copy;
1542 if (PREDICT_TRUE (n_left >= 2))
1544 vlib_prefetch_buffer_with_index (vm, buffers[1], STORE);
1545 CLIB_PREFETCH (&n_left, sizeof (n_left), STORE);
1548 //Move from available to used ring
1549 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = desc_head;
1550 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = desc_len;
1551 vhost_user_log_dirty_ring (vui, rxvq,
1552 ring[rxvq->last_used_idx & qsz_mask]);
1554 rxvq->last_avail_idx++;
1555 rxvq->last_used_idx++;
1557 n_left--; //At the end for error counting when 'goto done' is invoked
1561 CLIB_MEMORY_BARRIER ();
1562 rxvq->used->idx = rxvq->last_used_idx;
1563 vhost_user_log_dirty_ring (vui, rxvq, idx);
1565 /* interrupt (call) handling */
1566 if ((rxvq->callfd > 0) && !(rxvq->avail->flags & 1))
1568 rxvq->n_since_last_int += n_packets - n_left;
1570 if (rxvq->n_since_last_int > vum->coalesce_frames)
1571 vhost_user_send_call (vm, rxvq);
1576 if (PREDICT_FALSE (vui->lockp != 0))
1579 if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
1581 vlib_error_count (vm, node->node_index, error, n_left);
1582 vlib_increment_simple_counter
1583 (vnet_main.interface_main.sw_if_counters
1584 + VNET_INTERFACE_COUNTER_DROP,
1585 os_get_cpu_number (), vui->sw_if_index, n_left);
1588 vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
1589 return frame->n_vectors;
1592 static clib_error_t *
1593 vhost_user_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
1596 vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
1597 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1598 vhost_user_main_t *vum = &vhost_user_main;
1599 vhost_user_intf_t *vui =
1600 vec_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance);
1602 vui->admin_up = is_up;
1605 vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
1606 VNET_HW_INTERFACE_FLAG_LINK_UP);
1608 return /* no error */ 0;
1612 VNET_DEVICE_CLASS (vhost_user_dev_class,static) = {
1613 .name = "vhost-user",
1614 .tx_function = vhost_user_intfc_tx,
1615 .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR,
1616 .tx_function_error_strings = vhost_user_tx_func_error_strings,
1617 .format_device_name = format_vhost_user_interface_name,
1618 .name_renumber = vhost_user_name_renumber,
1619 .admin_up_down_function = vhost_user_interface_admin_up_down,
1620 .no_flatten_output_chains = 1,
1623 VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class,
1624 vhost_user_intfc_tx)
1628 vhost_user_process (vlib_main_t * vm,
1629 vlib_node_runtime_t * rt, vlib_frame_t * f)
1631 vhost_user_main_t *vum = &vhost_user_main;
1632 vhost_user_intf_t *vui;
1633 struct sockaddr_un sun;
1635 unix_file_t template = { 0 };
1636 f64 timeout = 3153600000.0 /* 100 years */ ;
1637 uword *event_data = 0;
1639 sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
1640 sun.sun_family = AF_UNIX;
1641 template.read_function = vhost_user_socket_read;
1642 template.error_function = vhost_user_socket_error;
1650 vlib_process_wait_for_event_or_clock (vm, timeout);
1651 vlib_process_get_events (vm, &event_data);
1652 vec_reset_length (event_data);
1656 vec_foreach (vui, vum->vhost_user_interfaces)
1659 if (vui->sock_is_server || !vui->active)
1662 if (vui->unix_fd == -1)
1664 /* try to connect */
1666 strncpy (sun.sun_path, (char *) vui->sock_filename,
1667 sizeof (sun.sun_path) - 1);
1670 (sockfd, (struct sockaddr *) &sun,
1671 sizeof (struct sockaddr_un)) == 0)
1673 vui->sock_errno = 0;
1674 vui->unix_fd = sockfd;
1675 template.file_descriptor = sockfd;
1676 vui->unix_file_index = unix_file_add (&unix_main, &template);
1677 hash_set (vum->vhost_user_interface_index_by_sock_fd, sockfd,
1678 vui - vum->vhost_user_interfaces);
1680 sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
1686 vui->sock_errno = errno;
1691 /* check if socket is alive */
1693 socklen_t len = sizeof (error);
1695 getsockopt (vui->unix_fd, SOL_SOCKET, SO_ERROR, &error, &len);
1698 vhost_user_if_disconnect (vui);
1706 VLIB_REGISTER_NODE (vhost_user_process_node,static) = {
1707 .function = vhost_user_process,
1708 .type = VLIB_NODE_TYPE_PROCESS,
1709 .name = "vhost-user-process",
1714 vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm, u32 sw_if_index)
1716 vhost_user_main_t *vum = &vhost_user_main;
1717 vhost_user_intf_t *vui;
1721 p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
1724 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1728 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
1731 // interface is inactive
1733 // disconnect interface sockets
1734 vhost_user_if_disconnect (vui);
1735 // add to inactive interface list
1736 vec_add1 (vum->vhost_user_inactive_interfaces_index, p[0]);
1738 // reset renumbered iface
1739 if (p[0] < vec_len (vum->show_dev_instance_by_real_dev_instance))
1740 vum->show_dev_instance_by_real_dev_instance[p[0]] = ~0;
1742 ethernet_delete_interface (vnm, vui->hw_if_index);
1743 DBG_SOCK ("deleted (deactivated) vhost-user interface instance %d", p[0]);
1748 // init server socket on specified sock_filename
1750 vhost_user_init_server_sock (const char *sock_filename, int *sockfd)
1753 struct sockaddr_un un = { };
1755 /* create listening socket */
1756 fd = socket (AF_UNIX, SOCK_STREAM, 0);
1760 return VNET_API_ERROR_SYSCALL_ERROR_1;
1763 un.sun_family = AF_UNIX;
1764 strncpy ((char *) un.sun_path, (char *) sock_filename,
1765 sizeof (un.sun_path) - 1);
1767 /* remove if exists */
1768 unlink ((char *) sock_filename);
1770 if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1)
1772 rv = VNET_API_ERROR_SYSCALL_ERROR_2;
1776 if (listen (fd, 1) == -1)
1778 rv = VNET_API_ERROR_SYSCALL_ERROR_3;
1782 unix_file_t template = { 0 };
1783 template.read_function = vhost_user_socksvr_accept_ready;
1784 template.file_descriptor = fd;
1785 unix_file_add (&unix_main, &template);
1794 // get new vhost_user_intf_t from inactive interfaces or create new one
1795 static vhost_user_intf_t *
1796 vhost_user_vui_new ()
1798 vhost_user_main_t *vum = &vhost_user_main;
1799 vhost_user_intf_t *vui = NULL;
1800 int inactive_cnt = vec_len (vum->vhost_user_inactive_interfaces_index);
1801 // if there are any inactive ifaces
1802 if (inactive_cnt > 0)
1806 vum->vhost_user_inactive_interfaces_index[inactive_cnt - 1];
1807 if (vec_len (vum->vhost_user_interfaces) > vui_idx)
1809 vui = vec_elt_at_index (vum->vhost_user_interfaces, vui_idx);
1810 DBG_SOCK ("reusing inactive vhost-user interface index %d",
1813 // "remove" from inactive list
1814 _vec_len (vum->vhost_user_inactive_interfaces_index) -= 1;
1817 // vui was not retrieved from inactive ifaces - create new
1819 vec_add2 (vum->vhost_user_interfaces, vui, 1);
1823 // create ethernet interface for vhost user intf
1825 vhost_user_create_ethernet (vnet_main_t * vnm, vlib_main_t * vm,
1826 vhost_user_intf_t * vui, u8 * hwaddress)
1828 vhost_user_main_t *vum = &vhost_user_main;
1830 clib_error_t *error;
1832 /* create hw and sw interface */
1835 clib_memcpy (hwaddr, hwaddress, 6);
1839 f64 now = vlib_time_now (vm);
1841 rnd = (u32) (now * 1e6);
1842 rnd = random_u32 (&rnd);
1844 clib_memcpy (hwaddr + 2, &rnd, sizeof (rnd));
1849 error = ethernet_register_interface
1851 vhost_user_dev_class.index,
1852 vui - vum->vhost_user_interfaces /* device instance */ ,
1853 hwaddr /* ethernet address */ ,
1854 &vui->hw_if_index, 0 /* flag change */ );
1856 clib_error_report (error);
1858 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, vui->hw_if_index);
1859 hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = 9000;
1862 // initialize vui with specified attributes
1864 vhost_user_vui_init (vnet_main_t * vnm,
1865 vhost_user_intf_t * vui, int sockfd,
1866 const char *sock_filename,
1867 u8 is_server, u64 feature_mask, u32 * sw_if_index)
1869 vnet_sw_interface_t *sw;
1870 sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
1871 vlib_thread_main_t *tm = vlib_get_thread_main ();
1874 vui->unix_fd = sockfd;
1875 vui->sw_if_index = sw->sw_if_index;
1876 vui->num_vrings = 2;
1877 vui->sock_is_server = is_server;
1878 strncpy (vui->sock_filename, sock_filename,
1879 ARRAY_LEN (vui->sock_filename) - 1);
1880 vui->sock_errno = 0;
1882 vui->feature_mask = feature_mask;
1884 vui->unix_file_index = ~0;
1885 vui->log_base_addr = 0;
1887 for (q = 0; q < 2; q++)
1889 vui->vrings[q].enabled = 0;
1892 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
1895 *sw_if_index = vui->sw_if_index;
1897 if (tm->n_vlib_mains > 1)
1899 vui->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
1900 CLIB_CACHE_LINE_BYTES);
1901 memset ((void *) vui->lockp, 0, CLIB_CACHE_LINE_BYTES);
1905 // register vui and start polling on it
1907 vhost_user_vui_register (vlib_main_t * vm, vhost_user_intf_t * vui)
1909 vhost_user_main_t *vum = &vhost_user_main;
1911 dpdk_main_t *dm = &dpdk_main;
1913 vlib_thread_main_t *tm = vlib_get_thread_main ();
1916 hash_set (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd,
1917 vui - vum->vhost_user_interfaces);
1918 hash_set (vum->vhost_user_interface_index_by_sw_if_index, vui->sw_if_index,
1919 vui - vum->vhost_user_interfaces);
1923 cpu_index = dm->input_cpu_first_index +
1924 (vui - vum->vhost_user_interfaces) % dm->input_cpu_count;
1926 if (tm->n_vlib_mains == 1)
1928 vlib_node_set_state (vm, vhost_user_input_node.index,
1929 VLIB_NODE_STATE_POLLING);
1932 vlib_node_set_state (vlib_mains[cpu_index], vhost_user_input_node.index,
1933 VLIB_NODE_STATE_POLLING);
1936 /* tell process to start polling for sockets */
1937 vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0);
1941 vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm,
1942 const char *sock_filename,
1946 u8 renumber, u32 custom_dev_instance, u8 * hwaddr)
1948 vhost_user_intf_t *vui = NULL;
1955 if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
1961 vui = vhost_user_vui_new ();
1962 ASSERT (vui != NULL);
1964 vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
1965 vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
1966 feature_mask, &sw_if_idx);
1970 vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
1973 vhost_user_vui_register (vm, vui);
1976 *sw_if_index = sw_if_idx;
1982 vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm,
1983 const char *sock_filename,
1986 u64 feature_mask, u8 renumber, u32 custom_dev_instance)
1988 vhost_user_main_t *vum = &vhost_user_main;
1989 vhost_user_intf_t *vui = NULL;
1995 p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
1998 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
2002 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
2005 // interface is inactive
2007 // disconnect interface sockets
2008 vhost_user_if_disconnect (vui);
2012 if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
2018 vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
2019 feature_mask, &sw_if_idx);
2023 vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
2026 vhost_user_vui_register (vm, vui);
2032 vhost_user_connect_command_fn (vlib_main_t * vm,
2033 unformat_input_t * input,
2034 vlib_cli_command_t * cmd)
2036 unformat_input_t _line_input, *line_input = &_line_input;
2037 u8 *sock_filename = NULL;
2040 u64 feature_mask = (u64) ~ 0;
2042 u32 custom_dev_instance = ~0;
2046 /* Get a line of input. */
2047 if (!unformat_user (input, unformat_line_input, line_input))
2050 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2052 if (unformat (line_input, "socket %s", &sock_filename))
2054 else if (unformat (line_input, "server"))
2056 else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
2060 (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
2062 else if (unformat (line_input, "renumber %d", &custom_dev_instance))
2067 return clib_error_return (0, "unknown input `%U'",
2068 format_unformat_error, input);
2070 unformat_free (line_input);
2072 vnet_main_t *vnm = vnet_get_main ();
2075 if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename,
2076 is_server, &sw_if_index, feature_mask,
2077 renumber, custom_dev_instance, hw)))
2079 vec_free (sock_filename);
2080 return clib_error_return (0, "vhost_user_create_if returned %d", rv);
2083 vec_free (sock_filename);
2084 vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
2090 vhost_user_delete_command_fn (vlib_main_t * vm,
2091 unformat_input_t * input,
2092 vlib_cli_command_t * cmd)
2094 unformat_input_t _line_input, *line_input = &_line_input;
2095 u32 sw_if_index = ~0;
2097 /* Get a line of input. */
2098 if (!unformat_user (input, unformat_line_input, line_input))
2101 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2103 if (unformat (line_input, "sw_if_index %d", &sw_if_index))
2106 return clib_error_return (0, "unknown input `%U'",
2107 format_unformat_error, input);
2109 unformat_free (line_input);
2111 vnet_main_t *vnm = vnet_get_main ();
2113 vhost_user_delete_if (vnm, vm, sw_if_index);
2119 vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm,
2120 vhost_user_intf_details_t ** out_vuids)
2123 vhost_user_main_t *vum = &vhost_user_main;
2124 vhost_user_intf_t *vui;
2125 vhost_user_intf_details_t *r_vuids = NULL;
2126 vhost_user_intf_details_t *vuid = NULL;
2127 u32 *hw_if_indices = 0;
2128 vnet_hw_interface_t *hi;
2135 vec_foreach (vui, vum->vhost_user_interfaces)
2138 vec_add1 (hw_if_indices, vui->hw_if_index);
2141 for (i = 0; i < vec_len (hw_if_indices); i++)
2143 hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
2144 vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);
2146 vec_add2 (r_vuids, vuid, 1);
2147 vuid->sw_if_index = vui->sw_if_index;
2148 vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz;
2149 vuid->features = vui->features;
2150 vuid->is_server = vui->sock_is_server;
2151 vuid->num_regions = vui->nregions;
2152 vuid->sock_errno = vui->sock_errno;
2153 strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename,
2154 ARRAY_LEN (vuid->sock_filename) - 1);
2156 s = format (s, "%v%c", hi->name, 0);
2158 strncpy ((char *) vuid->if_name, (char *) s,
2159 ARRAY_LEN (vuid->if_name) - 1);
2164 vec_free (hw_if_indices);
2166 *out_vuids = r_vuids;
2172 show_vhost_user_command_fn (vlib_main_t * vm,
2173 unformat_input_t * input,
2174 vlib_cli_command_t * cmd)
2176 clib_error_t *error = 0;
2177 vnet_main_t *vnm = vnet_get_main ();
2178 vhost_user_main_t *vum = &vhost_user_main;
2179 vhost_user_intf_t *vui;
2180 u32 hw_if_index, *hw_if_indices = 0;
2181 vnet_hw_interface_t *hi;
2189 struct feat_struct *feat_entry;
2191 static struct feat_struct feat_array[] = {
2192 #define _(s,b) { .str = #s, .bit = b, },
2193 foreach_virtio_net_feature
2198 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2201 (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
2203 vec_add1 (hw_if_indices, hw_if_index);
2204 vlib_cli_output (vm, "add %d", hw_if_index);
2206 else if (unformat (input, "descriptors") || unformat (input, "desc"))
2210 error = clib_error_return (0, "unknown input `%U'",
2211 format_unformat_error, input);
2215 if (vec_len (hw_if_indices) == 0)
2217 vec_foreach (vui, vum->vhost_user_interfaces)
2220 vec_add1 (hw_if_indices, vui->hw_if_index);
2223 vlib_cli_output (vm, "Virtio vhost-user interfaces");
2224 vlib_cli_output (vm, "Global:\n coalesce frames %d time %e\n\n",
2225 vum->coalesce_frames, vum->coalesce_time);
2227 for (i = 0; i < vec_len (hw_if_indices); i++)
2229 hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
2230 vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);
2231 vlib_cli_output (vm, "Interface: %s (ifindex %d)",
2232 hi->name, hw_if_indices[i]);
2234 vlib_cli_output (vm, "virtio_net_hdr_sz %d\n features (0x%llx): \n",
2235 vui->virtio_net_hdr_sz, vui->features);
2237 feat_entry = (struct feat_struct *) &feat_array;
2238 while (feat_entry->str)
2240 if (vui->features & (1 << feat_entry->bit))
2241 vlib_cli_output (vm, " %s (%d)", feat_entry->str,
2246 vlib_cli_output (vm, "\n");
2249 vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n",
2251 vui->sock_is_server ? "server" : "client",
2252 strerror (vui->sock_errno));
2254 vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions);
2258 vlib_cli_output (vm,
2259 " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n");
2260 vlib_cli_output (vm,
2261 " ====== ===== ================== ================== ================== ================== ==================\n");
2263 for (j = 0; j < vui->nregions; j++)
2265 vlib_cli_output (vm,
2266 " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
2267 j, vui->region_mmap_fd[j],
2268 vui->regions[j].guest_phys_addr,
2269 vui->regions[j].memory_size,
2270 vui->regions[j].userspace_addr,
2271 vui->regions[j].mmap_offset,
2272 pointer_to_uword (vui->region_mmap_addr[j]));
2274 for (q = 0; q < vui->num_vrings; q++)
2276 vlib_cli_output (vm, "\n Virtqueue %d\n", q);
2278 vlib_cli_output (vm,
2279 " qsz %d last_avail_idx %d last_used_idx %d\n",
2280 vui->vrings[q].qsz, vui->vrings[q].last_avail_idx,
2281 vui->vrings[q].last_used_idx);
2283 if (vui->vrings[q].avail && vui->vrings[q].used)
2284 vlib_cli_output (vm,
2285 " avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
2286 vui->vrings[q].avail->flags,
2287 vui->vrings[q].avail->idx,
2288 vui->vrings[q].used->flags,
2289 vui->vrings[q].used->idx);
2291 vlib_cli_output (vm, " kickfd %d callfd %d errfd %d\n",
2292 vui->vrings[q].kickfd,
2293 vui->vrings[q].callfd, vui->vrings[q].errfd);
2297 vlib_cli_output (vm, "\n descriptor table:\n");
2298 vlib_cli_output (vm,
2299 " id addr len flags next user_addr\n");
2300 vlib_cli_output (vm,
2301 " ===== ================== ===== ====== ===== ==================\n");
2302 for (j = 0; j < vui->vrings[q].qsz; j++)
2304 vlib_cli_output (vm,
2305 " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
2306 j, vui->vrings[q].desc[j].addr,
2307 vui->vrings[q].desc[j].len,
2308 vui->vrings[q].desc[j].flags,
2309 vui->vrings[q].desc[j].next,
2310 pointer_to_uword (map_guest_mem
2312 vui->vrings[q].desc[j].
2317 vlib_cli_output (vm, "\n");
2320 vec_free (hw_if_indices);
2330 VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
2331 .path = "create vhost-user",
2332 .short_help = "create vhost-user socket <socket-filename> [server] [feature-mask <hex>] [renumber <dev_instance>]",
2333 .function = vhost_user_connect_command_fn,
2336 VLIB_CLI_COMMAND (vhost_user_delete_command, static) = {
2337 .path = "delete vhost-user",
2338 .short_help = "delete vhost-user sw_if_index <nn>",
2339 .function = vhost_user_delete_command_fn,
2342 VLIB_CLI_COMMAND (show_vhost_user_command, static) = {
2343 .path = "show vhost-user",
2344 .short_help = "show vhost-user interface",
2345 .function = show_vhost_user_command_fn,
2350 static clib_error_t *
2351 vhost_user_config (vlib_main_t * vm, unformat_input_t * input)
2353 vhost_user_main_t *vum = &vhost_user_main;
2355 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2357 if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames))
2359 else if (unformat (input, "coalesce-time %f", &vum->coalesce_time))
2361 else if (unformat (input, "dont-dump-memory"))
2362 vum->dont_dump_vhost_user_memory = 1;
2364 return clib_error_return (0, "unknown input `%U'",
2365 format_unformat_error, input);
2371 /* vhost-user { ... } configuration. */
2372 VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user");
2375 vhost_user_unmap_all (void)
2377 vhost_user_main_t *vum = &vhost_user_main;
2378 vhost_user_intf_t *vui;
2380 if (vum->dont_dump_vhost_user_memory)
2382 vec_foreach (vui, vum->vhost_user_interfaces)
2384 unmap_all_mem_regions (vui);
2390 * fd.io coding-style-patch-verification: ON
2393 * eval: (c-set-style "gnu")