2 *------------------------------------------------------------------
5 * Copyright (c) 2014 Cisco and/or its affiliates.
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at:
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *------------------------------------------------------------------
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
36 #include <vnet/ip/ip.h>
38 #include <vnet/ethernet/ethernet.h>
39 #include <vnet/devices/devices.h>
40 #include <vnet/feature/feature.h>
42 #include <vnet/devices/virtio/vhost-user.h>
44 #define VHOST_USER_DEBUG_SOCKET 0
45 #define VHOST_USER_DEBUG_VQ 0
47 /* Set to get virtio_net_hdr in buffer pre-data
48 details will be shown in packet trace */
49 #define VHOST_USER_COPY_TX_HDR 0
51 #if VHOST_USER_DEBUG_SOCKET == 1
52 #define DBG_SOCK(args...) clib_warning(args);
54 #define DBG_SOCK(args...)
57 #if VHOST_USER_DEBUG_VQ == 1
58 #define DBG_VQ(args...) clib_warning(args);
60 #define DBG_VQ(args...)
63 vlib_node_registration_t vhost_user_input_node;
65 #define foreach_vhost_user_tx_func_error \
67 _(NOT_READY, "vhost user state error") \
68 _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)") \
69 _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)") \
70 _(MMAP_FAIL, "mmap failure") \
71 _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")
75 #define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f,
76 foreach_vhost_user_tx_func_error
78 VHOST_USER_TX_FUNC_N_ERROR,
79 } vhost_user_tx_func_error_t;
81 static char *vhost_user_tx_func_error_strings[] = {
83 foreach_vhost_user_tx_func_error
87 #define foreach_vhost_user_input_func_error \
88 _(NO_ERROR, "no error") \
89 _(NO_BUFFER, "no available buffer") \
90 _(MMAP_FAIL, "mmap failure") \
91 _(INDIRECT_OVERFLOW, "indirect descriptor overflows table") \
92 _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
93 _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")
97 #define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
98 foreach_vhost_user_input_func_error
100 VHOST_USER_INPUT_FUNC_N_ERROR,
101 } vhost_user_input_func_error_t;
103 static char *vhost_user_input_func_error_strings[] = {
105 foreach_vhost_user_input_func_error
110 static vhost_user_main_t vhost_user_main = {
114 VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = {
115 .name = "vhost-user",
120 format_vhost_user_interface_name (u8 * s, va_list * args)
122 u32 i = va_arg (*args, u32);
123 u32 show_dev_instance = ~0;
124 vhost_user_main_t *vum = &vhost_user_main;
126 if (i < vec_len (vum->show_dev_instance_by_real_dev_instance))
127 show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i];
129 if (show_dev_instance != ~0)
130 i = show_dev_instance;
132 s = format (s, "VirtualEthernet0/0/%d", i);
137 vhost_user_name_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance)
139 vhost_user_main_t *vum = &vhost_user_main;
141 vec_validate_init_empty (vum->show_dev_instance_by_real_dev_instance,
142 hi->dev_instance, ~0);
144 vum->show_dev_instance_by_real_dev_instance[hi->dev_instance] =
147 DBG_SOCK ("renumbered vhost-user interface dev_instance %d to %d",
148 hi->dev_instance, new_dev_instance);
153 static_always_inline void *
154 map_guest_mem (vhost_user_intf_t * vui, uword addr, u32 * hint)
157 if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) &&
158 ((vui->regions[i].guest_phys_addr +
159 vui->regions[i].memory_size) > addr)))
161 return (void *) (vui->region_mmap_addr[i] + addr -
162 vui->regions[i].guest_phys_addr);
165 __m128i rl, rh, al, ah, r;
166 al = _mm_set1_epi64x (addr + 1);
167 ah = _mm_set1_epi64x (addr);
169 rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
170 rl = _mm_cmpgt_epi64 (al, rl);
171 rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
172 rh = _mm_cmpgt_epi64 (rh, ah);
173 r = _mm_and_si128 (rl, rh);
175 rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
176 rl = _mm_cmpgt_epi64 (al, rl);
177 rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
178 rh = _mm_cmpgt_epi64 (rh, ah);
179 r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
181 rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
182 rl = _mm_cmpgt_epi64 (al, rl);
183 rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
184 rh = _mm_cmpgt_epi64 (rh, ah);
185 r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
187 rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
188 rl = _mm_cmpgt_epi64 (al, rl);
189 rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
190 rh = _mm_cmpgt_epi64 (rh, ah);
191 r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
193 r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
194 i = __builtin_ctzll (_mm_movemask_epi8 (r));
196 if (i < vui->nregions)
199 return (void *) (vui->region_mmap_addr[i] + addr -
200 vui->regions[i].guest_phys_addr);
204 for (i = 0; i < vui->nregions; i++)
206 if ((vui->regions[i].guest_phys_addr <= addr) &&
207 ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
211 return (void *) (vui->region_mmap_addr[i] + addr -
212 vui->regions[i].guest_phys_addr);
216 DBG_VQ ("failed to map guest mem addr %llx", addr);
222 map_user_mem (vhost_user_intf_t * vui, uword addr)
225 for (i = 0; i < vui->nregions; i++)
227 if ((vui->regions[i].userspace_addr <= addr) &&
228 ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) >
231 return (void *) (vui->region_mmap_addr[i] + addr -
232 vui->regions[i].userspace_addr);
239 get_huge_page_size (int fd)
247 unmap_all_mem_regions (vhost_user_intf_t * vui)
250 for (i = 0; i < vui->nregions; i++)
252 if (vui->region_mmap_addr[i] != (void *) -1)
255 long page_sz = get_huge_page_size (vui->region_mmap_fd[i]);
257 ssize_t map_sz = (vui->regions[i].memory_size +
258 vui->regions[i].mmap_offset +
259 page_sz) & ~(page_sz - 1);
262 munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset,
266 ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i,
267 vui->region_mmap_addr[i], map_sz, page_sz);
269 vui->region_mmap_addr[i] = (void *) -1;
273 clib_warning ("failed to unmap memory region (errno %d)",
276 close (vui->region_mmap_fd[i]);
283 static clib_error_t *
284 vhost_user_callfd_read_ready (unix_file_t * uf)
286 __attribute__ ((unused)) int n;
288 n = read (uf->file_descriptor, ((char *) &buff), 8);
293 vhost_user_if_disconnect (vhost_user_intf_t * vui)
295 vhost_user_main_t *vum = &vhost_user_main;
296 vnet_main_t *vnm = vnet_get_main ();
299 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
301 if (vui->unix_file_index != ~0)
303 unix_file_del (&unix_main, unix_main.file_pool + vui->unix_file_index);
304 vui->unix_file_index = ~0;
307 close (vui->unix_fd);
309 hash_unset (vum->vhost_user_interface_index_by_sock_fd, vui->unix_fd);
310 hash_unset (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd);
313 for (q = 0; q < vui->num_vrings; q++)
315 if (vui->vrings[q].callfd > -1)
317 unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
318 vui->vrings[q].callfd_idx);
319 unix_file_del (&unix_main, uf);
322 if (vui->vrings[q].kickfd > -1)
323 close (vui->vrings[q].kickfd);
325 vui->vrings[q].callfd = -1;
326 vui->vrings[q].kickfd = -1;
327 vui->vrings[q].desc = NULL;
328 vui->vrings[q].avail = NULL;
329 vui->vrings[q].used = NULL;
330 vui->vrings[q].log_guest_addr = 0;
331 vui->vrings[q].log_used = 0;
334 unmap_all_mem_regions (vui);
335 DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index);
338 #define VHOST_LOG_PAGE 0x1000
340 vhost_user_log_dirty_pages (vhost_user_intf_t * vui, u64 addr, u64 len)
342 if (PREDICT_TRUE (vui->log_base_addr == 0
343 || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL))))
347 if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size))
349 DBG_SOCK ("vhost_user_log_dirty_pages(): out of range\n");
353 CLIB_MEMORY_BARRIER ();
354 u64 page = addr / VHOST_LOG_PAGE;
355 while (page * VHOST_LOG_PAGE < addr + len)
357 ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8;
362 #define vhost_user_log_dirty_ring(vui, vq, member) \
363 if (PREDICT_FALSE(vq->log_used)) { \
364 vhost_user_log_dirty_pages(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \
365 sizeof(vq->used->member)); \
368 static clib_error_t *
369 vhost_user_socket_read (unix_file_t * uf)
372 int fd, number_of_fds = 0;
373 int fds[VHOST_MEMORY_MAX_NREGIONS];
374 vhost_user_msg_t msg;
377 vhost_user_main_t *vum = &vhost_user_main;
378 vhost_user_intf_t *vui;
379 struct cmsghdr *cmsg;
382 unix_file_t template = { 0 };
383 vnet_main_t *vnm = vnet_get_main ();
385 p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
386 uf->file_descriptor);
389 DBG_SOCK ("FD %d doesn't belong to any interface", uf->file_descriptor);
393 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
395 char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))];
397 memset (&mh, 0, sizeof (mh));
398 memset (control, 0, sizeof (control));
400 for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++)
403 /* set the payload */
404 iov[0].iov_base = (void *) &msg;
405 iov[0].iov_len = VHOST_USER_MSG_HDR_SZ;
409 mh.msg_control = control;
410 mh.msg_controllen = sizeof (control);
412 n = recvmsg (uf->file_descriptor, &mh, 0);
414 if (n != VHOST_USER_MSG_HDR_SZ)
417 if (mh.msg_flags & MSG_CTRUNC)
422 cmsg = CMSG_FIRSTHDR (&mh);
424 if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
425 (cmsg->cmsg_type == SCM_RIGHTS) &&
426 (cmsg->cmsg_len - CMSG_LEN (0) <=
427 VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
429 number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
430 clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
433 /* version 1, no reply bit set */
434 if ((msg.flags & 7) != 1)
436 DBG_SOCK ("malformed message received. closing socket");
441 int rv __attribute__ ((unused));
442 /* $$$$ pay attention to rv */
443 rv = read (uf->file_descriptor, ((char *) &msg) + n, msg.size);
448 case VHOST_USER_GET_FEATURES:
449 DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES", vui->hw_if_index);
452 msg.u64 = (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
453 (1 << FEAT_VIRTIO_F_ANY_LAYOUT) |
454 (1 << FEAT_VIRTIO_F_INDIRECT_DESC) |
455 (1 << FEAT_VHOST_F_LOG_ALL) |
456 (1 << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
457 (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
458 (1UL << FEAT_VIRTIO_F_VERSION_1);
459 msg.u64 &= vui->feature_mask;
461 msg.size = sizeof (msg.u64);
464 case VHOST_USER_SET_FEATURES:
465 DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx",
466 vui->hw_if_index, msg.u64);
468 vui->features = msg.u64;
470 if (vui->features & (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF))
471 vui->virtio_net_hdr_sz = 12;
473 vui->virtio_net_hdr_sz = 10;
476 (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0;
478 ASSERT (vui->virtio_net_hdr_sz < VLIB_BUFFER_PRE_DATA_SIZE);
479 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
482 for (q = 0; q < 2; q++)
484 vui->vrings[q].desc = 0;
485 vui->vrings[q].avail = 0;
486 vui->vrings[q].used = 0;
487 vui->vrings[q].log_guest_addr = 0;
488 vui->vrings[q].log_used = 0;
491 DBG_SOCK ("interface %d disconnected", vui->sw_if_index);
495 case VHOST_USER_SET_MEM_TABLE:
496 DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
497 vui->hw_if_index, msg.memory.nregions);
499 if ((msg.memory.nregions < 1) ||
500 (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
503 DBG_SOCK ("number of mem regions must be between 1 and %i",
504 VHOST_MEMORY_MAX_NREGIONS);
509 if (msg.memory.nregions != number_of_fds)
511 DBG_SOCK ("each memory region must have FD");
514 unmap_all_mem_regions (vui);
515 for (i = 0; i < msg.memory.nregions; i++)
517 clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
518 sizeof (vhost_user_memory_region_t));
520 long page_sz = get_huge_page_size (fds[i]);
522 /* align size to 2M page */
523 ssize_t map_sz = (vui->regions[i].memory_size +
524 vui->regions[i].mmap_offset +
525 page_sz) & ~(page_sz - 1);
527 vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
528 MAP_SHARED, fds[i], 0);
529 vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
530 vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
531 vui->regions[i].memory_size;
534 ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
535 "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i],
538 if (vui->region_mmap_addr[i] == MAP_FAILED)
540 clib_warning ("failed to map memory. errno is %d", errno);
543 vui->region_mmap_addr[i] += vui->regions[i].mmap_offset;
544 vui->region_mmap_fd[i] = fds[i];
546 vui->nregions = msg.memory.nregions;
549 case VHOST_USER_SET_VRING_NUM:
550 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
551 vui->hw_if_index, msg.state.index, msg.state.num);
553 if ((msg.state.num > 32768) || /* maximum ring size is 32768 */
554 (msg.state.num == 0) || /* it cannot be zero */
555 (msg.state.num % 2)) /* must be power of 2 */
557 vui->vrings[msg.state.index].qsz = msg.state.num;
560 case VHOST_USER_SET_VRING_ADDR:
561 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
562 vui->hw_if_index, msg.state.index);
564 vui->vrings[msg.state.index].desc = (vring_desc_t *)
565 map_user_mem (vui, msg.addr.desc_user_addr);
566 vui->vrings[msg.state.index].used = (vring_used_t *)
567 map_user_mem (vui, msg.addr.used_user_addr);
568 vui->vrings[msg.state.index].avail = (vring_avail_t *)
569 map_user_mem (vui, msg.addr.avail_user_addr);
571 if ((vui->vrings[msg.state.index].desc == NULL) ||
572 (vui->vrings[msg.state.index].used == NULL) ||
573 (vui->vrings[msg.state.index].avail == NULL))
575 DBG_SOCK ("failed to map user memory for hw_if_index %d",
580 vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
581 vui->vrings[msg.state.index].log_used =
582 (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;
584 /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
585 the ring is initialized in an enabled state. */
587 if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES)))
589 vui->vrings[msg.state.index].enabled = 1;
592 vui->vrings[msg.state.index].last_used_idx =
593 vui->vrings[msg.state.index].last_avail_idx =
594 vui->vrings[msg.state.index].used->idx;
596 /* tell driver that we don't want interrupts */
597 vui->vrings[msg.state.index].used->flags |= 1;
600 case VHOST_USER_SET_OWNER:
601 DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index);
604 case VHOST_USER_RESET_OWNER:
605 DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index);
608 case VHOST_USER_SET_VRING_CALL:
609 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL u64 %d",
610 vui->hw_if_index, msg.u64);
612 q = (u8) (msg.u64 & 0xFF);
614 if (!(msg.u64 & 0x100))
616 if (number_of_fds != 1)
619 /* if there is old fd, delete it */
620 if (vui->vrings[q].callfd > -1)
622 unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
623 vui->vrings[q].callfd_idx);
624 unix_file_del (&unix_main, uf);
626 vui->vrings[q].callfd = fds[0];
627 template.read_function = vhost_user_callfd_read_ready;
628 template.file_descriptor = fds[0];
629 vui->vrings[q].callfd_idx = unix_file_add (&unix_main, &template);
632 vui->vrings[q].callfd = -1;
635 case VHOST_USER_SET_VRING_KICK:
636 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK u64 %d",
637 vui->hw_if_index, msg.u64);
639 q = (u8) (msg.u64 & 0xFF);
641 if (!(msg.u64 & 0x100))
643 if (number_of_fds != 1)
646 if (vui->vrings[q].kickfd > -1)
647 close (vui->vrings[q].kickfd);
649 vui->vrings[q].kickfd = fds[0];
652 vui->vrings[q].kickfd = -1;
655 case VHOST_USER_SET_VRING_ERR:
656 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR u64 %d",
657 vui->hw_if_index, msg.u64);
659 q = (u8) (msg.u64 & 0xFF);
661 if (!(msg.u64 & 0x100))
663 if (number_of_fds != 1)
671 vui->vrings[q].errfd = fd;
674 case VHOST_USER_SET_VRING_BASE:
675 DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d",
676 vui->hw_if_index, msg.state.index, msg.state.num);
678 vui->vrings[msg.state.index].last_avail_idx = msg.state.num;
681 case VHOST_USER_GET_VRING_BASE:
682 DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
683 vui->hw_if_index, msg.state.index, msg.state.num);
685 /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
686 vui->vrings[msg.state.index].enabled = 0;
688 msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
690 msg.size = sizeof (msg.state);
693 case VHOST_USER_NONE:
694 DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index);
698 case VHOST_USER_SET_LOG_BASE:
700 DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index);
702 if (msg.size != sizeof (msg.log))
705 ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
706 msg.size, sizeof (msg.log));
711 (vui->protocol_features & (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD)))
714 ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
719 /* align size to 2M page */
720 long page_sz = get_huge_page_size (fd);
722 (msg.log.size + msg.log.offset + page_sz) & ~(page_sz - 1);
724 vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
728 ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
729 map_sz, msg.log.offset, fd, vui->log_base_addr);
731 if (vui->log_base_addr == MAP_FAILED)
733 clib_warning ("failed to map memory. errno is %d", errno);
737 vui->log_base_addr += msg.log.offset;
738 vui->log_size = msg.log.size;
741 msg.size = sizeof (msg.u64);
746 case VHOST_USER_SET_LOG_FD:
747 DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index);
751 case VHOST_USER_GET_PROTOCOL_FEATURES:
752 DBG_SOCK ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES",
756 msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
757 msg.size = sizeof (msg.u64);
760 case VHOST_USER_SET_PROTOCOL_FEATURES:
761 DBG_SOCK ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%lx",
762 vui->hw_if_index, msg.u64);
764 vui->protocol_features = msg.u64;
768 case VHOST_USER_SET_VRING_ENABLE:
769 DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE, enable: %d",
770 vui->hw_if_index, msg.state.num);
771 vui->vrings[msg.state.index].enabled = msg.state.num;
775 DBG_SOCK ("unknown vhost-user message %d received. closing socket",
780 /* if we have pointers to descriptor table, go up */
782 vui->vrings[VHOST_NET_VRING_IDX_TX].desc &&
783 vui->vrings[VHOST_NET_VRING_IDX_RX].desc)
786 DBG_SOCK ("interface %d connected", vui->sw_if_index);
788 vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
789 VNET_HW_INTERFACE_FLAG_LINK_UP);
794 /* if we need to reply */
798 send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0);
799 if (n != (msg.size + VHOST_USER_MSG_HDR_SZ))
806 vhost_user_if_disconnect (vui);
810 static clib_error_t *
811 vhost_user_socket_error (unix_file_t * uf)
813 vhost_user_main_t *vum = &vhost_user_main;
814 vhost_user_intf_t *vui;
817 p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
818 uf->file_descriptor);
821 DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
825 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
827 vhost_user_if_disconnect (vui);
831 static clib_error_t *
832 vhost_user_socksvr_accept_ready (unix_file_t * uf)
834 int client_fd, client_len;
835 struct sockaddr_un client;
836 unix_file_t template = { 0 };
837 vhost_user_main_t *vum = &vhost_user_main;
838 vhost_user_intf_t *vui;
841 p = hash_get (vum->vhost_user_interface_index_by_listener_fd,
842 uf->file_descriptor);
845 DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
849 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
851 client_len = sizeof (client);
852 client_fd = accept (uf->file_descriptor,
853 (struct sockaddr *) &client,
854 (socklen_t *) & client_len);
857 return clib_error_return_unix (0, "accept");
859 template.read_function = vhost_user_socket_read;
860 template.error_function = vhost_user_socket_error;
861 template.file_descriptor = client_fd;
862 vui->unix_file_index = unix_file_add (&unix_main, &template);
864 vui->client_fd = client_fd;
865 hash_set (vum->vhost_user_interface_index_by_sock_fd, vui->client_fd,
866 vui - vum->vhost_user_interfaces);
871 static clib_error_t *
872 vhost_user_init (vlib_main_t * vm)
875 vhost_user_main_t *vum = &vhost_user_main;
876 vlib_thread_main_t *tm = vlib_get_thread_main ();
877 vlib_thread_registration_t *tr;
880 error = vlib_call_init_function (vm, ip4_init);
884 vum->vhost_user_interface_index_by_listener_fd =
885 hash_create (0, sizeof (uword));
886 vum->vhost_user_interface_index_by_sock_fd =
887 hash_create (0, sizeof (uword));
888 vum->vhost_user_interface_index_by_sw_if_index =
889 hash_create (0, sizeof (uword));
890 vum->coalesce_frames = 32;
891 vum->coalesce_time = 1e-3;
893 vec_validate_aligned (vum->rx_buffers, tm->n_vlib_mains - 1,
894 CLIB_CACHE_LINE_BYTES);
896 /* find out which cpus will be used for input */
897 vum->input_cpu_first_index = 0;
898 vum->input_cpu_count = 1;
899 p = hash_get_mem (tm->thread_registrations_by_name, "workers");
900 tr = p ? (vlib_thread_registration_t *) p[0] : 0;
902 if (tr && tr->count > 0)
904 vum->input_cpu_first_index = tr->first_index;
905 vum->input_cpu_count = tr->count;
911 VLIB_INIT_FUNCTION (vhost_user_init);
913 static clib_error_t *
914 vhost_user_exit (vlib_main_t * vm)
920 VLIB_MAIN_LOOP_EXIT_FUNCTION (vhost_user_exit);
926 #if VHOST_USER_COPY_TX_HDR == 1
927 virtio_net_hdr_t hdr;
929 } vhost_user_input_trace_t;
932 format_vhost_user_input_trace (u8 * s, va_list * va)
934 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
935 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
936 CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
937 vhost_user_main_t *vum = &vhost_user_main;
938 vhost_user_input_trace_t *t = va_arg (*va, vhost_user_input_trace_t *);
939 vhost_user_intf_t *vui = vec_elt_at_index (vum->vhost_user_interfaces,
942 vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, vui->sw_if_index);
944 #if VHOST_USER_COPY_TX_HDR == 1
945 uword indent = format_get_indent (s);
948 s = format (s, "%U virtqueue %d",
949 format_vnet_sw_interface_name, vnm, sw, t->virtqueue);
951 #if VHOST_USER_COPY_TX_HDR == 1
952 s = format (s, "\n%Uvirtio_net_hdr flags 0x%02x gso_type %u hdr_len %u",
953 format_white_space, indent,
954 t->hdr.flags, t->hdr.gso_type, t->hdr.hdr_len);
961 vhost_user_rx_trace (vlib_main_t * vm,
962 vlib_node_runtime_t * node,
963 vhost_user_intf_t * vui, i16 virtqueue)
966 vhost_user_main_t *vum = &vhost_user_main;
968 u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
970 n_left = vec_len (vui->d_trace_buffers);
971 b = vui->d_trace_buffers;
977 vhost_user_input_trace_t *t0;
982 b0 = vlib_get_buffer (vm, bi0);
983 vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 0);
984 t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
985 t0->virtqueue = virtqueue;
986 t0->device_index = vui - vum->vhost_user_interfaces;
987 #if VHOST_USER_COPY_TX_HDR == 1
988 clib_memcpy (&t0->hdr, b0->pre_data, sizeof (virtio_net_hdr_t));
996 vhost_user_send_call (vlib_main_t * vm, vhost_user_vring_t * vq)
998 vhost_user_main_t *vum = &vhost_user_main;
1000 int rv __attribute__ ((unused));
1001 /* $$$$ pay attention to rv */
1002 rv = write (vq->callfd, &x, sizeof (x));
1003 vq->n_since_last_int = 0;
1004 vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time;
1009 vhost_user_if_input (vlib_main_t * vm,
1010 vhost_user_main_t * vum,
1011 vhost_user_intf_t * vui, vlib_node_runtime_t * node)
1013 vhost_user_vring_t *txvq = &vui->vrings[VHOST_NET_VRING_IDX_TX];
1014 vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
1015 uword n_rx_packets = 0, n_rx_bytes = 0;
1017 u32 n_left_to_next, *to_next;
1020 uword n_trace = vlib_get_trace_count (vm, node);
1022 u32 cpu_index, rx_len, drops, flush;
1023 f64 now = vlib_time_now (vm);
1024 u32 map_guest_hint_desc = 0;
1025 u32 map_guest_hint_indirect = 0;
1026 u32 *map_guest_hint_p = &map_guest_hint_desc;
1028 vec_reset_length (vui->d_trace_buffers);
1030 /* no descriptor ptr - bail out */
1031 if (PREDICT_FALSE (!txvq->desc || !txvq->avail || !txvq->enabled))
1034 /* do we have pending intterupts ? */
1035 if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
1036 vhost_user_send_call (vm, txvq);
1038 if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
1039 vhost_user_send_call (vm, rxvq);
1041 /* only bit 0 of avail.flags is used so we don't want to deal with this
1042 interface if any other bit is set */
1043 if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
1046 n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);
1049 if (PREDICT_FALSE (n_left == 0))
1052 if (PREDICT_FALSE (n_left == txvq->qsz))
1054 //Informational error logging when VPP is not receiving packets fast enough
1055 vlib_error_count (vm, node->node_index,
1056 VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
1059 if (PREDICT_FALSE (!vui->admin_up))
1061 /* if intf is admin down, just drop all packets waiting in the ring */
1062 txvq->last_avail_idx = txvq->last_used_idx = txvq->avail->idx;
1063 CLIB_MEMORY_BARRIER ();
1064 txvq->used->idx = txvq->last_used_idx;
1065 vhost_user_log_dirty_ring (vui, txvq, idx);
1066 vhost_user_send_call (vm, txvq);
1070 qsz_mask = txvq->qsz - 1;
1071 cpu_index = os_get_cpu_number ();
1075 if (n_left > VLIB_FRAME_SIZE)
1076 n_left = VLIB_FRAME_SIZE;
1078 /* Allocate some buffers.
1079 * Note that buffers that are chained for jumbo
1080 * frames are allocated separately using a slower path.
1081 * The idea is to be certain to have enough buffers at least
1082 * to cycle through the descriptors without having to check for errors.
1083 * For jumbo frames, the bottleneck is memory copy anyway.
1085 if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
1087 vec_alloc (vum->rx_buffers[cpu_index], 2 * VLIB_FRAME_SIZE);
1089 if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
1090 flush = n_left; //Drop all input
1093 if (PREDICT_FALSE (_vec_len (vum->rx_buffers[cpu_index]) < n_left))
1095 u32 curr_len = _vec_len (vum->rx_buffers[cpu_index]);
1096 _vec_len (vum->rx_buffers[cpu_index]) +=
1097 vlib_buffer_alloc_from_free_list (vm,
1098 vum->rx_buffers[cpu_index] +
1100 2 * VLIB_FRAME_SIZE - curr_len,
1101 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1103 if (PREDICT_FALSE (n_left > _vec_len (vum->rx_buffers[cpu_index])))
1104 flush = n_left - _vec_len (vum->rx_buffers[cpu_index]);
1107 if (PREDICT_FALSE (flush))
1109 //Remove some input buffers
1112 vlib_error_count (vm, vhost_user_input_node.index,
1113 VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
1116 u16 desc_chain_head =
1117 txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
1118 txvq->last_avail_idx++;
1119 txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
1121 txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
1122 vhost_user_log_dirty_ring (vui, txvq,
1123 ring[txvq->last_used_idx & qsz_mask]);
1124 txvq->last_used_idx++;
1129 rx_len = vec_len (vum->rx_buffers[cpu_index]); //vector might be null
1132 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1134 while (n_left > 0 && n_left_to_next > 0)
1136 vlib_buffer_t *b_head, *b_current;
1137 u32 bi_head, bi_current;
1138 u16 desc_chain_head, desc_current;
1139 u8 error = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;
1141 if (PREDICT_TRUE (n_left > 1))
1144 txvq->avail->ring[(txvq->last_avail_idx + 1) & qsz_mask];
1146 map_guest_mem (vui, txvq->desc[next_desc].addr,
1147 &map_guest_hint_desc);
1148 if (PREDICT_TRUE (buffer_addr != 0))
1149 CLIB_PREFETCH (buffer_addr, 64, STORE);
1151 u32 bi = vum->rx_buffers[cpu_index][rx_len - 2];
1152 vlib_prefetch_buffer_with_index (vm, bi, STORE);
1153 CLIB_PREFETCH (vlib_get_buffer (vm, bi)->data, 128, STORE);
1156 desc_chain_head = desc_current =
1157 txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
1158 bi_head = bi_current = vum->rx_buffers[cpu_index][--rx_len];
1159 b_head = b_current = vlib_get_buffer (vm, bi_head);
1160 vlib_buffer_chain_init (b_head);
1163 if (PREDICT_TRUE (vui->is_any_layout) ||
1164 (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
1165 !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)))
1167 /* ANYLAYOUT or single buffer */
1168 offset = vui->virtio_net_hdr_sz;
1172 /* CSR case without ANYLAYOUT, skip 1st buffer */
1173 offset = txvq->desc[desc_current].len;
1176 vring_desc_t *desc_table = txvq->desc;
1177 u32 desc_index = desc_current;
1178 map_guest_hint_p = &map_guest_hint_desc;
1180 if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
1182 desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
1183 &map_guest_hint_desc);
1185 map_guest_hint_p = &map_guest_hint_indirect;
1186 if (PREDICT_FALSE (desc_table == 0))
1188 error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
1196 map_guest_mem (vui, desc_table[desc_index].addr,
1198 if (PREDICT_FALSE (buffer_addr == 0))
1200 error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
1205 (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
1207 CLIB_PREFETCH (&desc_table[desc_table[desc_index].next],
1208 sizeof (vring_desc_t), STORE);
1211 #if VHOST_USER_COPY_TX_HDR == 1
1212 if (PREDICT_TRUE (offset))
1213 clib_memcpy (b->pre_data, buffer_addr, sizeof (virtio_net_hdr_t)); /* 12 byte hdr is not used on tx */
1216 if (desc_table[desc_index].len > offset)
1218 u16 len = desc_table[desc_index].len - offset;
1219 u16 copied = vlib_buffer_chain_append_data_with_alloc (vm,
1220 VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX,
1229 error = VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER;
1235 /* if next flag is set, take next desc in the chain */
1236 if ((desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
1237 desc_index = desc_table[desc_index].next;
1243 /* consume the descriptor and return it as used */
1244 txvq->last_avail_idx++;
1245 txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
1247 txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
1248 vhost_user_log_dirty_ring (vui, txvq,
1249 ring[txvq->last_used_idx & qsz_mask]);
1250 txvq->last_used_idx++;
1252 //It is important to free RX as fast as possible such that the TX
1253 //process does not drop packets
1254 if ((txvq->last_used_idx & 0x3f) == 0) // Every 64 packets
1255 txvq->used->idx = txvq->last_used_idx;
1257 if (PREDICT_FALSE (b_head->current_length < 14 &&
1258 error == VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
1259 error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;
1261 VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);
1263 vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
1264 vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
1265 b_head->error = node->errors[error];
1267 if (PREDICT_FALSE (n_trace > n_rx_packets))
1268 vec_add1 (vui->d_trace_buffers, bi_head);
1270 if (PREDICT_FALSE (error))
1273 next0 = VNET_DEVICE_INPUT_NEXT_DROP;
1278 b_head->current_length +
1279 b_head->total_length_not_including_first_buffer;
1281 next0 = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
1284 to_next[0] = bi_head;
1288 /* redirect if feature path enabled */
1289 vnet_feature_device_input_redirect_x1 (node, vui->sw_if_index,
1292 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1293 to_next, n_left_to_next,
1298 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1302 if (PREDICT_TRUE (vum->rx_buffers[cpu_index] != 0))
1303 _vec_len (vum->rx_buffers[cpu_index]) = rx_len;
1305 /* give buffers back to driver */
1306 CLIB_MEMORY_BARRIER ();
1307 txvq->used->idx = txvq->last_used_idx;
1308 vhost_user_log_dirty_ring (vui, txvq, idx);
1310 if (PREDICT_FALSE (vec_len (vui->d_trace_buffers) > 0))
1312 vhost_user_rx_trace (vm, node, vui, VHOST_NET_VRING_IDX_TX);
1313 vlib_set_trace_count (vm, node,
1314 n_trace - vec_len (vui->d_trace_buffers));
1317 /* interrupt (call) handling */
1318 if ((txvq->callfd > -1) && !(txvq->avail->flags & 1))
1320 txvq->n_since_last_int += n_rx_packets;
1322 if (txvq->n_since_last_int > vum->coalesce_frames)
1323 vhost_user_send_call (vm, txvq);
1326 if (PREDICT_FALSE (drops))
1328 vlib_increment_simple_counter
1329 (vnet_main.interface_main.sw_if_counters
1330 + VNET_INTERFACE_COUNTER_DROP, os_get_cpu_number (),
1331 vui->sw_if_index, drops);
1334 /* increase rx counters */
1335 vlib_increment_combined_counter
1336 (vnet_main.interface_main.combined_sw_if_counters
1337 + VNET_INTERFACE_COUNTER_RX,
1338 os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes);
1340 return n_rx_packets;
1344 vhost_user_input (vlib_main_t * vm,
1345 vlib_node_runtime_t * node, vlib_frame_t * f)
1347 vhost_user_main_t *vum = &vhost_user_main;
1348 u32 cpu_index = os_get_cpu_number ();
1349 vhost_user_intf_t *vui;
1350 uword n_rx_packets = 0;
1353 for (i = 0; i < vec_len (vum->vhost_user_interfaces); i++)
1355 vui = vec_elt_at_index (vum->vhost_user_interfaces, i);
1358 if ((i % vum->input_cpu_count) ==
1359 (cpu_index - vum->input_cpu_first_index))
1360 n_rx_packets += vhost_user_if_input (vm, vum, vui, node);
1363 return n_rx_packets;
1367 VLIB_REGISTER_NODE (vhost_user_input_node) = {
1368 .function = vhost_user_input,
1369 .type = VLIB_NODE_TYPE_INPUT,
1370 .name = "vhost-user-input",
1372 /* Will be enabled if/when hardware is detected. */
1373 .state = VLIB_NODE_STATE_DISABLED,
1375 .format_buffer = format_ethernet_header_with_length,
1376 .format_trace = format_vhost_user_input_trace,
1378 .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
1379 .error_strings = vhost_user_input_func_error_strings,
1381 .n_next_nodes = VNET_DEVICE_INPUT_N_NEXT_NODES,
1382 .next_nodes = VNET_DEVICE_INPUT_NEXT_NODES,
1385 VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input)
1389 vhost_user_intfc_tx (vlib_main_t * vm,
1390 vlib_node_runtime_t * node, vlib_frame_t * frame)
1392 u32 *buffers = vlib_frame_args (frame);
1394 vhost_user_main_t *vum = &vhost_user_main;
1395 uword n_packets = 0;
1396 vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
1397 vhost_user_intf_t *vui =
1398 vec_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
1399 vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
1401 u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;
1403 n_left = n_packets = frame->n_vectors;
1404 u32 map_guest_hint_desc = 0;
1405 u32 map_guest_hint_indirect = 0;
1406 u32 *map_guest_hint_p = &map_guest_hint_desc;
1408 if (PREDICT_FALSE (!vui->is_up))
1412 (!rxvq->desc || !rxvq->avail || vui->sock_errno != 0 || !rxvq->enabled))
1414 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
1418 if (PREDICT_FALSE (vui->lockp != 0))
1420 while (__sync_lock_test_and_set (vui->lockp, 1))
1424 /* only bit 0 of avail.flags is used so we don't want to deal with this
1425 interface if any other bit is set */
1426 if (PREDICT_FALSE (rxvq->avail->flags & 0xFFFE))
1428 error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
1432 if (PREDICT_FALSE ((rxvq->avail->idx == rxvq->last_avail_idx)))
1434 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1438 qsz_mask = rxvq->qsz - 1; /* qsz is always power of 2 */
1442 vlib_buffer_t *b0, *current_b0;
1443 u16 desc_head, desc_index, desc_len;
1444 vring_desc_t *desc_table;
1448 b0 = vlib_get_buffer (vm, buffers[0]);
1451 if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
1453 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1457 desc_table = rxvq->desc;
1458 map_guest_hint_p = &map_guest_hint_desc;
1459 desc_head = desc_index =
1460 rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
1461 if (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)
1464 (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
1466 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
1471 map_guest_mem (vui, rxvq->desc[desc_index].addr,
1472 &map_guest_hint_desc))))
1474 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1478 map_guest_hint_p = &map_guest_hint_indirect;
1481 desc_len = vui->virtio_net_hdr_sz;
1485 map_guest_mem (vui, desc_table[desc_index].addr,
1486 map_guest_hint_p))))
1488 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1491 buffer_len = desc_table[desc_index].len;
1493 CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);
1495 virtio_net_hdr_mrg_rxbuf_t *hdr =
1496 (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
1498 hdr->hdr.gso_type = 0;
1499 if (vui->virtio_net_hdr_sz == 12)
1500 hdr->num_buffers = 1;
1502 vhost_user_log_dirty_pages (vui, desc_table[desc_index].addr,
1503 vui->virtio_net_hdr_sz);
1505 u16 bytes_left = b0->current_length;
1506 buffer_addr += vui->virtio_net_hdr_sz;
1507 buffer_len -= vui->virtio_net_hdr_sz;
1513 if (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT)
1515 current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
1516 bytes_left = current_b0->current_length;
1525 if (buffer_len == 0)
1527 if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT)
1529 //Next one is chained
1530 desc_index = desc_table[desc_index].next;
1533 map_guest_mem (vui, desc_table[desc_index].addr,
1534 map_guest_hint_p))))
1536 rxvq->last_used_idx -= hdr->num_buffers - 1;
1537 rxvq->last_avail_idx -= hdr->num_buffers - 1;
1538 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1541 buffer_len = desc_table[desc_index].len;
1543 else if (vui->virtio_net_hdr_sz == 12) //MRG is available
1545 //Move from available to used buffer
1546 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id =
1548 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len =
1550 vhost_user_log_dirty_ring (vui, rxvq,
1551 ring[rxvq->last_used_idx &
1553 rxvq->last_avail_idx++;
1554 rxvq->last_used_idx++;
1558 (rxvq->last_avail_idx == rxvq->avail->idx))
1560 //Dequeue queued descriptors for this packet
1561 rxvq->last_used_idx -= hdr->num_buffers - 1;
1562 rxvq->last_avail_idx -= hdr->num_buffers - 1;
1563 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
1567 desc_table = rxvq->desc;
1568 map_guest_hint_p = &map_guest_hint_desc;
1569 desc_head = desc_index =
1570 rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
1572 (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
1574 //It is seriously unlikely that a driver will put indirect descriptor
1575 //after non-indirect descriptor.
1577 (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
1579 error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
1585 rxvq->desc[desc_index].addr,
1586 &map_guest_hint_desc))))
1588 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1592 map_guest_hint_p = &map_guest_hint_indirect;
1597 map_guest_mem (vui, desc_table[desc_index].addr,
1598 map_guest_hint_p))))
1600 error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
1603 buffer_len = desc_table[desc_index].len;
1604 CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);
1608 error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
1613 u16 bytes_to_copy = bytes_left;
1615 (bytes_to_copy > buffer_len) ? buffer_len : bytes_to_copy;
1616 clib_memcpy (buffer_addr,
1617 vlib_buffer_get_current (current_b0) +
1618 current_b0->current_length - bytes_left,
1621 vhost_user_log_dirty_pages (vui,
1622 desc_table[desc_index].addr +
1623 desc_table[desc_index].len -
1624 bytes_left - bytes_to_copy,
1627 bytes_left -= bytes_to_copy;
1628 buffer_len -= bytes_to_copy;
1629 buffer_addr += bytes_to_copy;
1630 desc_len += bytes_to_copy;
1633 //Move from available to used ring
1634 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = desc_head;
1635 rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = desc_len;
1636 vhost_user_log_dirty_ring (vui, rxvq,
1637 ring[rxvq->last_used_idx & qsz_mask]);
1639 rxvq->last_avail_idx++;
1640 rxvq->last_used_idx++;
1642 n_left--; //At the end for error counting when 'goto done' is invoked
1646 CLIB_MEMORY_BARRIER ();
1647 rxvq->used->idx = rxvq->last_used_idx;
1648 vhost_user_log_dirty_ring (vui, rxvq, idx);
1650 /* interrupt (call) handling */
1651 if ((rxvq->callfd > -1) && !(rxvq->avail->flags & 1))
1653 rxvq->n_since_last_int += n_packets - n_left;
1655 if (rxvq->n_since_last_int > vum->coalesce_frames)
1656 vhost_user_send_call (vm, rxvq);
1661 if (PREDICT_FALSE (vui->lockp != 0))
1664 if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
1666 vlib_error_count (vm, node->node_index, error, n_left);
1667 vlib_increment_simple_counter
1668 (vnet_main.interface_main.sw_if_counters
1669 + VNET_INTERFACE_COUNTER_DROP,
1670 os_get_cpu_number (), vui->sw_if_index, n_left);
1673 vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
1674 return frame->n_vectors;
1677 static clib_error_t *
1678 vhost_user_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
1681 vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
1682 uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1683 vhost_user_main_t *vum = &vhost_user_main;
1684 vhost_user_intf_t *vui =
1685 vec_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance);
1687 vui->admin_up = is_up;
1690 vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
1691 VNET_HW_INTERFACE_FLAG_LINK_UP);
1693 return /* no error */ 0;
1697 VNET_DEVICE_CLASS (vhost_user_dev_class,static) = {
1698 .name = "vhost-user",
1699 .tx_function = vhost_user_intfc_tx,
1700 .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR,
1701 .tx_function_error_strings = vhost_user_tx_func_error_strings,
1702 .format_device_name = format_vhost_user_interface_name,
1703 .name_renumber = vhost_user_name_renumber,
1704 .admin_up_down_function = vhost_user_interface_admin_up_down,
1705 .no_flatten_output_chains = 1,
1708 VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class,
1709 vhost_user_intfc_tx)
1713 vhost_user_process (vlib_main_t * vm,
1714 vlib_node_runtime_t * rt, vlib_frame_t * f)
1716 vhost_user_main_t *vum = &vhost_user_main;
1717 vhost_user_intf_t *vui;
1718 struct sockaddr_un sun;
1720 unix_file_t template = { 0 };
1721 f64 timeout = 3153600000.0 /* 100 years */ ;
1722 uword *event_data = 0;
1724 sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
1725 sun.sun_family = AF_UNIX;
1726 template.read_function = vhost_user_socket_read;
1727 template.error_function = vhost_user_socket_error;
1735 vlib_process_wait_for_event_or_clock (vm, timeout);
1736 vlib_process_get_events (vm, &event_data);
1737 vec_reset_length (event_data);
1741 vec_foreach (vui, vum->vhost_user_interfaces)
1744 if (vui->sock_is_server || !vui->active)
1747 if (vui->unix_fd == -1)
1749 /* try to connect */
1751 strncpy (sun.sun_path, (char *) vui->sock_filename,
1752 sizeof (sun.sun_path) - 1);
1755 (sockfd, (struct sockaddr *) &sun,
1756 sizeof (struct sockaddr_un)) == 0)
1758 vui->sock_errno = 0;
1759 vui->unix_fd = sockfd;
1760 template.file_descriptor = sockfd;
1761 vui->unix_file_index = unix_file_add (&unix_main, &template);
1762 hash_set (vum->vhost_user_interface_index_by_sock_fd, sockfd,
1763 vui - vum->vhost_user_interfaces);
1765 sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
1771 vui->sock_errno = errno;
1776 /* check if socket is alive */
1778 socklen_t len = sizeof (error);
1780 getsockopt (vui->unix_fd, SOL_SOCKET, SO_ERROR, &error, &len);
1783 vhost_user_if_disconnect (vui);
1791 VLIB_REGISTER_NODE (vhost_user_process_node,static) = {
1792 .function = vhost_user_process,
1793 .type = VLIB_NODE_TYPE_PROCESS,
1794 .name = "vhost-user-process",
1799 vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm, u32 sw_if_index)
1801 vhost_user_main_t *vum = &vhost_user_main;
1802 vhost_user_intf_t *vui;
1806 p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
1809 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1813 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
1816 // interface is inactive
1818 // disconnect interface sockets
1819 vhost_user_if_disconnect (vui);
1820 // add to inactive interface list
1821 vec_add1 (vum->vhost_user_inactive_interfaces_index, p[0]);
1823 // reset renumbered iface
1824 if (p[0] < vec_len (vum->show_dev_instance_by_real_dev_instance))
1825 vum->show_dev_instance_by_real_dev_instance[p[0]] = ~0;
1827 ethernet_delete_interface (vnm, vui->hw_if_index);
1828 DBG_SOCK ("deleted (deactivated) vhost-user interface instance %d", p[0]);
1833 // init server socket on specified sock_filename
1835 vhost_user_init_server_sock (const char *sock_filename, int *sockfd)
1838 struct sockaddr_un un = { };
1840 /* create listening socket */
1841 fd = socket (AF_UNIX, SOCK_STREAM, 0);
1845 return VNET_API_ERROR_SYSCALL_ERROR_1;
1848 un.sun_family = AF_UNIX;
1849 strncpy ((char *) un.sun_path, (char *) sock_filename,
1850 sizeof (un.sun_path) - 1);
1852 /* remove if exists */
1853 unlink ((char *) sock_filename);
1855 if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1)
1857 rv = VNET_API_ERROR_SYSCALL_ERROR_2;
1861 if (listen (fd, 1) == -1)
1863 rv = VNET_API_ERROR_SYSCALL_ERROR_3;
1867 unix_file_t template = { 0 };
1868 template.read_function = vhost_user_socksvr_accept_ready;
1869 template.file_descriptor = fd;
1870 unix_file_add (&unix_main, &template);
1879 // get new vhost_user_intf_t from inactive interfaces or create new one
1880 static vhost_user_intf_t *
1881 vhost_user_vui_new ()
1883 vhost_user_main_t *vum = &vhost_user_main;
1884 vhost_user_intf_t *vui = NULL;
1885 int inactive_cnt = vec_len (vum->vhost_user_inactive_interfaces_index);
1886 // if there are any inactive ifaces
1887 if (inactive_cnt > 0)
1891 vum->vhost_user_inactive_interfaces_index[inactive_cnt - 1];
1892 if (vec_len (vum->vhost_user_interfaces) > vui_idx)
1894 vui = vec_elt_at_index (vum->vhost_user_interfaces, vui_idx);
1895 DBG_SOCK ("reusing inactive vhost-user interface index %d",
1898 // "remove" from inactive list
1899 _vec_len (vum->vhost_user_inactive_interfaces_index) -= 1;
1902 // vui was not retrieved from inactive ifaces - create new
1904 vec_add2 (vum->vhost_user_interfaces, vui, 1);
1908 // create ethernet interface for vhost user intf
1910 vhost_user_create_ethernet (vnet_main_t * vnm, vlib_main_t * vm,
1911 vhost_user_intf_t * vui, u8 * hwaddress)
1913 vhost_user_main_t *vum = &vhost_user_main;
1915 clib_error_t *error;
1917 /* create hw and sw interface */
1920 clib_memcpy (hwaddr, hwaddress, 6);
1924 f64 now = vlib_time_now (vm);
1926 rnd = (u32) (now * 1e6);
1927 rnd = random_u32 (&rnd);
1929 clib_memcpy (hwaddr + 2, &rnd, sizeof (rnd));
1934 error = ethernet_register_interface
1936 vhost_user_dev_class.index,
1937 vui - vum->vhost_user_interfaces /* device instance */ ,
1938 hwaddr /* ethernet address */ ,
1939 &vui->hw_if_index, 0 /* flag change */ );
1941 clib_error_report (error);
1943 vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, vui->hw_if_index);
1944 hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = 9000;
1947 // initialize vui with specified attributes
1949 vhost_user_vui_init (vnet_main_t * vnm,
1950 vhost_user_intf_t * vui, int sockfd,
1951 const char *sock_filename,
1952 u8 is_server, u64 feature_mask, u32 * sw_if_index)
1954 vnet_sw_interface_t *sw;
1955 sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
1956 vlib_thread_main_t *tm = vlib_get_thread_main ();
1959 vui->unix_fd = sockfd;
1960 vui->sw_if_index = sw->sw_if_index;
1961 vui->num_vrings = 2;
1962 vui->sock_is_server = is_server;
1963 strncpy (vui->sock_filename, sock_filename,
1964 ARRAY_LEN (vui->sock_filename) - 1);
1965 vui->sock_errno = 0;
1967 vui->feature_mask = feature_mask;
1969 vui->unix_file_index = ~0;
1970 vui->log_base_addr = 0;
1972 for (q = 0; q < 2; q++)
1974 vui->vrings[q].enabled = 0;
1975 vui->vrings[q].callfd = -1;
1976 vui->vrings[q].kickfd = -1;
1979 vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
1982 *sw_if_index = vui->sw_if_index;
1984 if (tm->n_vlib_mains > 1)
1986 vui->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
1987 CLIB_CACHE_LINE_BYTES);
1988 memset ((void *) vui->lockp, 0, CLIB_CACHE_LINE_BYTES);
1992 // register vui and start polling on it
1994 vhost_user_vui_register (vlib_main_t * vm, vhost_user_intf_t * vui)
1996 vhost_user_main_t *vum = &vhost_user_main;
1998 vlib_thread_main_t *tm = vlib_get_thread_main ();
2000 hash_set (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd,
2001 vui - vum->vhost_user_interfaces);
2002 hash_set (vum->vhost_user_interface_index_by_sw_if_index, vui->sw_if_index,
2003 vui - vum->vhost_user_interfaces);
2006 cpu_index = vum->input_cpu_first_index +
2007 (vui - vum->vhost_user_interfaces) % vum->input_cpu_count;
2009 if (tm->n_vlib_mains == 1)
2010 vlib_node_set_state (vm, vhost_user_input_node.index,
2011 VLIB_NODE_STATE_POLLING);
2013 vlib_node_set_state (vlib_mains[cpu_index], vhost_user_input_node.index,
2014 VLIB_NODE_STATE_POLLING);
2016 /* tell process to start polling for sockets */
2017 vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0);
2021 vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm,
2022 const char *sock_filename,
2026 u8 renumber, u32 custom_dev_instance, u8 * hwaddr)
2028 vhost_user_intf_t *vui = NULL;
2035 if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
2041 vui = vhost_user_vui_new ();
2042 ASSERT (vui != NULL);
2044 vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
2045 vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
2046 feature_mask, &sw_if_idx);
2050 vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
2053 vhost_user_vui_register (vm, vui);
2056 *sw_if_index = sw_if_idx;
2062 vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm,
2063 const char *sock_filename,
2066 u64 feature_mask, u8 renumber, u32 custom_dev_instance)
2068 vhost_user_main_t *vum = &vhost_user_main;
2069 vhost_user_intf_t *vui = NULL;
2075 p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
2078 return VNET_API_ERROR_INVALID_SW_IF_INDEX;
2082 vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
2085 // interface is inactive
2087 // disconnect interface sockets
2088 vhost_user_if_disconnect (vui);
2092 if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
2098 vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
2099 feature_mask, &sw_if_idx);
2103 vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
2106 vhost_user_vui_register (vm, vui);
2112 vhost_user_connect_command_fn (vlib_main_t * vm,
2113 unformat_input_t * input,
2114 vlib_cli_command_t * cmd)
2116 unformat_input_t _line_input, *line_input = &_line_input;
2117 u8 *sock_filename = NULL;
2120 u64 feature_mask = (u64) ~ 0;
2122 u32 custom_dev_instance = ~0;
2126 /* Get a line of input. */
2127 if (!unformat_user (input, unformat_line_input, line_input))
2130 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2132 if (unformat (line_input, "socket %s", &sock_filename))
2134 else if (unformat (line_input, "server"))
2136 else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
2140 (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
2142 else if (unformat (line_input, "renumber %d", &custom_dev_instance))
2147 return clib_error_return (0, "unknown input `%U'",
2148 format_unformat_error, input);
2150 unformat_free (line_input);
2152 vnet_main_t *vnm = vnet_get_main ();
2155 if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename,
2156 is_server, &sw_if_index, feature_mask,
2157 renumber, custom_dev_instance, hw)))
2159 vec_free (sock_filename);
2160 return clib_error_return (0, "vhost_user_create_if returned %d", rv);
2163 vec_free (sock_filename);
2164 vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
2170 vhost_user_delete_command_fn (vlib_main_t * vm,
2171 unformat_input_t * input,
2172 vlib_cli_command_t * cmd)
2174 unformat_input_t _line_input, *line_input = &_line_input;
2175 u32 sw_if_index = ~0;
2177 /* Get a line of input. */
2178 if (!unformat_user (input, unformat_line_input, line_input))
2181 while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2183 if (unformat (line_input, "sw_if_index %d", &sw_if_index))
2186 return clib_error_return (0, "unknown input `%U'",
2187 format_unformat_error, input);
2189 unformat_free (line_input);
2191 vnet_main_t *vnm = vnet_get_main ();
2193 vhost_user_delete_if (vnm, vm, sw_if_index);
2199 vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm,
2200 vhost_user_intf_details_t ** out_vuids)
2203 vhost_user_main_t *vum = &vhost_user_main;
2204 vhost_user_intf_t *vui;
2205 vhost_user_intf_details_t *r_vuids = NULL;
2206 vhost_user_intf_details_t *vuid = NULL;
2207 u32 *hw_if_indices = 0;
2208 vnet_hw_interface_t *hi;
2215 vec_foreach (vui, vum->vhost_user_interfaces)
2218 vec_add1 (hw_if_indices, vui->hw_if_index);
2221 for (i = 0; i < vec_len (hw_if_indices); i++)
2223 hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
2224 vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);
2226 vec_add2 (r_vuids, vuid, 1);
2227 vuid->sw_if_index = vui->sw_if_index;
2228 vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz;
2229 vuid->features = vui->features;
2230 vuid->is_server = vui->sock_is_server;
2231 vuid->num_regions = vui->nregions;
2232 vuid->sock_errno = vui->sock_errno;
2233 strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename,
2234 ARRAY_LEN (vuid->sock_filename) - 1);
2236 s = format (s, "%v%c", hi->name, 0);
2238 strncpy ((char *) vuid->if_name, (char *) s,
2239 ARRAY_LEN (vuid->if_name) - 1);
2244 vec_free (hw_if_indices);
2246 *out_vuids = r_vuids;
2252 show_vhost_user_command_fn (vlib_main_t * vm,
2253 unformat_input_t * input,
2254 vlib_cli_command_t * cmd)
2256 clib_error_t *error = 0;
2257 vnet_main_t *vnm = vnet_get_main ();
2258 vhost_user_main_t *vum = &vhost_user_main;
2259 vhost_user_intf_t *vui;
2260 u32 hw_if_index, *hw_if_indices = 0;
2261 vnet_hw_interface_t *hi;
2269 struct feat_struct *feat_entry;
2271 static struct feat_struct feat_array[] = {
2272 #define _(s,b) { .str = #s, .bit = b, },
2273 foreach_virtio_net_feature
2278 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2281 (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
2283 vec_add1 (hw_if_indices, hw_if_index);
2284 vlib_cli_output (vm, "add %d", hw_if_index);
2286 else if (unformat (input, "descriptors") || unformat (input, "desc"))
2290 error = clib_error_return (0, "unknown input `%U'",
2291 format_unformat_error, input);
2295 if (vec_len (hw_if_indices) == 0)
2297 vec_foreach (vui, vum->vhost_user_interfaces)
2300 vec_add1 (hw_if_indices, vui->hw_if_index);
2303 vlib_cli_output (vm, "Virtio vhost-user interfaces");
2304 vlib_cli_output (vm, "Global:\n coalesce frames %d time %e\n\n",
2305 vum->coalesce_frames, vum->coalesce_time);
2307 for (i = 0; i < vec_len (hw_if_indices); i++)
2309 hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
2310 vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);
2311 vlib_cli_output (vm, "Interface: %s (ifindex %d)",
2312 hi->name, hw_if_indices[i]);
2314 vlib_cli_output (vm, "virtio_net_hdr_sz %d\n features (0x%llx): \n",
2315 vui->virtio_net_hdr_sz, vui->features);
2317 feat_entry = (struct feat_struct *) &feat_array;
2318 while (feat_entry->str)
2320 if (vui->features & (1 << feat_entry->bit))
2321 vlib_cli_output (vm, " %s (%d)", feat_entry->str,
2326 vlib_cli_output (vm, "\n");
2329 vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n",
2331 vui->sock_is_server ? "server" : "client",
2332 strerror (vui->sock_errno));
2334 vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions);
2338 vlib_cli_output (vm,
2339 " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n");
2340 vlib_cli_output (vm,
2341 " ====== ===== ================== ================== ================== ================== ==================\n");
2343 for (j = 0; j < vui->nregions; j++)
2345 vlib_cli_output (vm,
2346 " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
2347 j, vui->region_mmap_fd[j],
2348 vui->regions[j].guest_phys_addr,
2349 vui->regions[j].memory_size,
2350 vui->regions[j].userspace_addr,
2351 vui->regions[j].mmap_offset,
2352 pointer_to_uword (vui->region_mmap_addr[j]));
2354 for (q = 0; q < vui->num_vrings; q++)
2356 vlib_cli_output (vm, "\n Virtqueue %d\n", q);
2358 vlib_cli_output (vm,
2359 " qsz %d last_avail_idx %d last_used_idx %d\n",
2360 vui->vrings[q].qsz, vui->vrings[q].last_avail_idx,
2361 vui->vrings[q].last_used_idx);
2363 if (vui->vrings[q].avail && vui->vrings[q].used)
2364 vlib_cli_output (vm,
2365 " avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
2366 vui->vrings[q].avail->flags,
2367 vui->vrings[q].avail->idx,
2368 vui->vrings[q].used->flags,
2369 vui->vrings[q].used->idx);
2371 vlib_cli_output (vm, " kickfd %d callfd %d errfd %d\n",
2372 vui->vrings[q].kickfd,
2373 vui->vrings[q].callfd, vui->vrings[q].errfd);
2377 vlib_cli_output (vm, "\n descriptor table:\n");
2378 vlib_cli_output (vm,
2379 " id addr len flags next user_addr\n");
2380 vlib_cli_output (vm,
2381 " ===== ================== ===== ====== ===== ==================\n");
2382 for (j = 0; j < vui->vrings[q].qsz; j++)
2385 vlib_cli_output (vm,
2386 " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
2387 j, vui->vrings[q].desc[j].addr,
2388 vui->vrings[q].desc[j].len,
2389 vui->vrings[q].desc[j].flags,
2390 vui->vrings[q].desc[j].next,
2391 pointer_to_uword (map_guest_mem
2393 vui->vrings[q].desc[j].
2398 vlib_cli_output (vm, "\n");
2401 vec_free (hw_if_indices);
2410 VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
2411 .path = "create vhost-user",
2412 .short_help = "create vhost-user socket <socket-filename> [server] [feature-mask <hex>] [renumber <dev_instance>]",
2413 .function = vhost_user_connect_command_fn,
2416 VLIB_CLI_COMMAND (vhost_user_delete_command, static) = {
2417 .path = "delete vhost-user",
2418 .short_help = "delete vhost-user sw_if_index <nn>",
2419 .function = vhost_user_delete_command_fn,
2422 VLIB_CLI_COMMAND (show_vhost_user_command, static) = {
2423 .path = "show vhost-user",
2424 .short_help = "show vhost-user interface",
2425 .function = show_vhost_user_command_fn,
2429 static clib_error_t *
2430 vhost_user_config (vlib_main_t * vm, unformat_input_t * input)
2432 vhost_user_main_t *vum = &vhost_user_main;
2434 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2436 if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames))
2438 else if (unformat (input, "coalesce-time %f", &vum->coalesce_time))
2440 else if (unformat (input, "dont-dump-memory"))
2441 vum->dont_dump_vhost_user_memory = 1;
2443 return clib_error_return (0, "unknown input `%U'",
2444 format_unformat_error, input);
2450 /* vhost-user { ... } configuration. */
2451 VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user");
2454 vhost_user_unmap_all (void)
2456 vhost_user_main_t *vum = &vhost_user_main;
2457 vhost_user_intf_t *vui;
2459 if (vum->dont_dump_vhost_user_memory)
2461 vec_foreach (vui, vum->vhost_user_interfaces)
2463 unmap_all_mem_regions (vui);
2469 * fd.io coding-style-patch-verification: ON
2472 * eval: (c-set-style "gnu")