2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/types.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
28 #include <sys/prctl.h>
29 #include <sys/eventfd.h>
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 #include <vnet/plugin/plugin.h>
36 #include <vnet/ethernet/ethernet.h>
37 #include <vpp/app/version.h>
39 #include <memif/memif.h>
40 #include <memif/private.h>
43 memif_str2vec (uint8_t * str, int len)
51 for (i = 0; i < len; i++)
63 memif_msg_send (int fd, memif_msg_t * msg, int afd)
65 struct msghdr mh = { 0 };
67 char ctl[CMSG_SPACE (sizeof (int))];
70 iov[0].iov_base = (void *) msg;
71 iov[0].iov_len = sizeof (memif_msg_t);
78 memset (&ctl, 0, sizeof (ctl));
80 mh.msg_controllen = sizeof (ctl);
81 cmsg = CMSG_FIRSTHDR (&mh);
82 cmsg->cmsg_len = CMSG_LEN (sizeof (int));
83 cmsg->cmsg_level = SOL_SOCKET;
84 cmsg->cmsg_type = SCM_RIGHTS;
85 memcpy (CMSG_DATA (cmsg), &afd, sizeof (int));
87 rv = sendmsg (fd, &mh, 0);
89 return clib_error_return_unix (0, "sendmsg");
90 DBG ("Message type %u sent (fd %d)", msg->type, afd);
95 memif_msg_enq_ack (memif_if_t * mif)
97 memif_msg_fifo_elt_t *e;
98 clib_fifo_add2 (mif->msg_queue, e);
100 e->msg.type = MEMIF_MSG_TYPE_ACK;
104 static clib_error_t *
105 memif_msg_enq_hello (int fd)
108 memif_msg_t msg = { 0 };
109 memif_msg_hello_t *h = &msg.hello;
110 msg.type = MEMIF_MSG_TYPE_HELLO;
111 h->min_version = MEMIF_VERSION;
112 h->max_version = MEMIF_VERSION;
113 h->max_m2s_ring = MEMIF_MAX_M2S_RING;
114 h->max_s2m_ring = MEMIF_MAX_M2S_RING;
115 h->max_region = MEMIF_MAX_REGION;
116 h->max_log2_ring_size = MEMIF_MAX_LOG2_RING_SIZE;
117 s = format (0, "VPP %s%c", VPP_BUILD_VER, 0);
118 strncpy ((char *) h->name, (char *) s, sizeof (h->name) - 1);
120 return memif_msg_send (fd, &msg, -1);
124 memif_msg_enq_init (memif_if_t * mif)
127 memif_msg_fifo_elt_t *e;
128 clib_fifo_add2 (mif->msg_queue, e);
129 memif_msg_init_t *i = &e->msg.init;
131 e->msg.type = MEMIF_MSG_TYPE_INIT;
133 i->version = MEMIF_VERSION;
136 s = format (0, "VPP %s%c", VPP_BUILD_VER, 0);
137 strncpy ((char *) i->name, (char *) s, sizeof (i->name) - 1);
139 strncpy ((char *) i->secret, (char *) mif->secret,
140 sizeof (i->secret) - 1);
145 memif_msg_enq_add_region (memif_if_t * mif, u8 region)
147 memif_msg_fifo_elt_t *e;
148 clib_fifo_add2 (mif->msg_queue, e);
149 memif_msg_add_region_t *ar = &e->msg.add_region;
151 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
152 e->fd = mif->regions[region].fd;
154 ar->size = mif->regions[region].region_size;
158 memif_msg_enq_add_ring (memif_if_t * mif, u8 index, u8 direction)
160 memif_msg_fifo_elt_t *e;
161 clib_fifo_add2 (mif->msg_queue, e);
162 memif_msg_add_ring_t *ar = &e->msg.add_ring;
165 ASSERT ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) != 0);
167 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
169 if (direction == MEMIF_RING_M2S)
170 mq = vec_elt_at_index (mif->rx_queues, index);
172 mq = vec_elt_at_index (mif->tx_queues, index);
176 ar->region = mq->region;
177 ar->offset = mq->offset;
178 ar->log2_ring_size = mq->log2_ring_size;
179 ar->flags = (direction == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
183 memif_msg_enq_connect (memif_if_t * mif)
185 memif_msg_fifo_elt_t *e;
186 clib_fifo_add2 (mif->msg_queue, e);
187 memif_msg_connect_t *c = &e->msg.connect;
190 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
192 s = format (0, "%U%c", format_memif_device_name, mif->dev_instance, 0);
193 strncpy ((char *) c->if_name, (char *) s, sizeof (c->if_name) - 1);
198 memif_msg_enq_connected (memif_if_t * mif)
200 memif_msg_fifo_elt_t *e;
201 clib_fifo_add2 (mif->msg_queue, e);
202 memif_msg_connected_t *c = &e->msg.connected;
205 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
207 s = format (0, "%U%c", format_memif_device_name, mif->dev_instance, 0);
208 strncpy ((char *) c->if_name, (char *) s, sizeof (c->if_name) - 1);
213 memif_msg_send_disconnect (memif_if_t * mif, clib_error_t * err)
215 memif_msg_t msg = { 0 };
216 msg.type = MEMIF_MSG_TYPE_DISCONNECT;
217 memif_msg_disconnect_t *d = &msg.disconnect;
220 strncpy ((char *) d->string, (char *) err->what, sizeof (d->string) - 1);
222 return memif_msg_send (mif->conn_fd, &msg, -1);
225 static clib_error_t *
226 memif_msg_receive_hello (memif_if_t * mif, memif_msg_t * msg)
228 memif_msg_hello_t *h = &msg->hello;
230 if (msg->hello.min_version > MEMIF_VERSION ||
231 msg->hello.max_version < MEMIF_VERSION)
232 return clib_error_return (0, "incompatible protocol version");
234 mif->run.num_s2m_rings = clib_min (h->max_s2m_ring + 1,
235 mif->cfg.num_s2m_rings);
236 mif->run.num_m2s_rings = clib_min (h->max_m2s_ring + 1,
237 mif->cfg.num_m2s_rings);
238 mif->run.log2_ring_size = clib_min (h->max_log2_ring_size,
239 mif->cfg.log2_ring_size);
240 mif->run.buffer_size = mif->cfg.buffer_size;
242 mif->remote_name = memif_str2vec (h->name, sizeof (h->name));
247 static clib_error_t *
248 memif_msg_receive_init (memif_if_t ** mifp, memif_msg_t * msg,
251 memif_main_t *mm = &memif_main;
252 memif_socket_file_t *msf =
253 vec_elt_at_index (mm->socket_files, uf->private_data);
254 memif_msg_init_t *i = &msg->init;
255 memif_if_t *mif, tmp;
259 if (i->version != MEMIF_VERSION)
261 memif_file_del_by_index (uf - unix_main.file_pool);
262 return clib_error_return (0, "unsupported version");
265 p = mhash_get (&msf->dev_instance_by_id, &i->id);
269 err = clib_error_return (0, "unmatched interface id");
273 mif = vec_elt_at_index (mm->interfaces, p[0]);
275 if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
277 err = clib_error_return (0, "cannot connect to slave");
281 if (mif->conn_fd != -1)
283 err = clib_error_return (0, "already connected");
287 if (i->mode != mif->mode)
289 err = clib_error_return (0, "mode mismatch");
293 mif->conn_fd = uf->file_descriptor;
294 mif->conn_unix_file_index = uf - unix_main.file_pool;
295 hash_set (msf->dev_instance_by_fd, mif->conn_fd, mif->dev_instance);
296 mif->remote_name = memif_str2vec (i->name, sizeof (i->name));
303 s = memif_str2vec (i->secret, sizeof (i->secret));
305 return clib_error_return (0, "secret required");
307 r = vec_cmp (s, mif->secret);
311 return clib_error_return (0, "incorrect secret");
317 tmp.conn_fd = uf->file_descriptor;
318 memif_msg_send_disconnect (&tmp, err);
319 memif_file_del_by_index (uf - unix_main.file_pool);
323 static clib_error_t *
324 memif_msg_receive_add_region (memif_if_t * mif, memif_msg_t * msg, int fd)
326 memif_msg_add_region_t *ar = &msg->add_region;
329 return clib_error_return (0, "missing memory region fd");
331 if (ar->index != vec_len (mif->regions))
332 return clib_error_return (0, "unexpected region index");
334 if (ar->index > MEMIF_MAX_REGION)
335 return clib_error_return (0, "too many regions");
337 vec_validate_aligned (mif->regions, ar->index, CLIB_CACHE_LINE_BYTES);
338 mr = vec_elt_at_index (mif->regions, ar->index);
340 mr->region_size = ar->size;
345 static clib_error_t *
346 memif_msg_receive_add_ring (memif_if_t * mif, memif_msg_t * msg, int fd)
348 memif_msg_add_ring_t *ar = &msg->add_ring;
352 return clib_error_return (0, "missing ring interrupt fd");
354 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M)
356 if (ar->index != vec_len (mif->rx_queues))
357 return clib_error_return (0, "unexpected ring index");
359 if (ar->index > MEMIF_MAX_S2M_RING)
360 return clib_error_return (0, "too many rings");
362 vec_validate_aligned (mif->rx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
363 mq = vec_elt_at_index (mif->rx_queues, ar->index);
364 mif->run.num_s2m_rings = vec_len (mif->rx_queues);
368 if (ar->index != vec_len (mif->tx_queues))
369 return clib_error_return (0, "unexpected ring index");
371 if (ar->index > MEMIF_MAX_M2S_RING)
372 return clib_error_return (0, "too many rings");
374 vec_validate_aligned (mif->tx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
375 mq = vec_elt_at_index (mif->tx_queues, ar->index);
376 mif->run.num_m2s_rings = vec_len (mif->tx_queues);
380 mq->int_unix_file_index = ~0;
381 mq->log2_ring_size = ar->log2_ring_size;
382 mq->region = ar->region;
383 mq->offset = ar->offset;
388 static clib_error_t *
389 memif_msg_receive_connect (memif_if_t * mif, memif_msg_t * msg)
392 memif_msg_connect_t *c = &msg->connect;
394 if ((err = memif_connect (mif)))
397 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
402 static clib_error_t *
403 memif_msg_receive_connected (memif_if_t * mif, memif_msg_t * msg)
406 memif_msg_connected_t *c = &msg->connected;
408 if ((err = memif_connect (mif)))
411 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
415 static clib_error_t *
416 memif_msg_receive_disconnect (memif_if_t * mif, memif_msg_t * msg)
418 memif_msg_disconnect_t *d = &msg->disconnect;
420 mif->remote_disc_string = memif_str2vec (d->string, sizeof (d->string));
421 return clib_error_return (0, "disconnect received");
424 static clib_error_t *
425 memif_msg_receive (memif_if_t ** mifp, unix_file_t * uf)
427 char ctl[CMSG_SPACE (sizeof (int)) +
428 CMSG_SPACE (sizeof (struct ucred))] = { 0 };
429 struct msghdr mh = { 0 };
431 memif_msg_t msg = { 0 };
433 clib_error_t *err = 0;
436 memif_if_t *mif = *mifp;
438 iov[0].iov_base = (void *) &msg;
439 iov[0].iov_len = sizeof (memif_msg_t);
442 mh.msg_control = ctl;
443 mh.msg_controllen = sizeof (ctl);
445 /* receive the incoming message */
446 size = recvmsg (uf->file_descriptor, &mh, 0);
447 if (size != sizeof (memif_msg_t))
449 return (size == 0) ? clib_error_return (0, "disconnected") :
450 clib_error_return_unix (0,
451 "recvmsg: malformed message received on fd %d",
452 uf->file_descriptor);
455 if (mif == 0 && msg.type != MEMIF_MSG_TYPE_INIT)
458 return clib_error_return (0, "unexpected message received");
461 /* process anciliary data */
462 struct ucred *cr = 0;
463 struct cmsghdr *cmsg;
465 cmsg = CMSG_FIRSTHDR (&mh);
468 if (cmsg->cmsg_level == SOL_SOCKET)
470 if (cmsg->cmsg_type == SCM_CREDENTIALS)
472 cr = (struct ucred *) CMSG_DATA (cmsg);
474 else if (cmsg->cmsg_type == SCM_RIGHTS)
476 int *fdp = (int *) CMSG_DATA (cmsg);
480 cmsg = CMSG_NXTHDR (&mh, cmsg);
483 DBG ("Message type %u received", msg.type);
484 /* process the message based on its type */
487 case MEMIF_MSG_TYPE_ACK:
490 case MEMIF_MSG_TYPE_HELLO:
491 if ((err = memif_msg_receive_hello (mif, &msg)))
493 if ((err = memif_init_regions_and_queues (mif)))
495 memif_msg_enq_init (mif);
496 memif_msg_enq_add_region (mif, 0);
497 vec_foreach_index (i, mif->tx_queues)
498 memif_msg_enq_add_ring (mif, i, MEMIF_RING_S2M);
499 vec_foreach_index (i, mif->rx_queues)
500 memif_msg_enq_add_ring (mif, i, MEMIF_RING_M2S);
501 memif_msg_enq_connect (mif);
504 case MEMIF_MSG_TYPE_INIT:
505 if ((err = memif_msg_receive_init (mifp, &msg, uf)))
508 mif->remote_pid = cr->pid;
509 mif->remote_uid = cr->uid;
510 mif->remote_gid = cr->gid;
511 memif_msg_enq_ack (mif);
514 case MEMIF_MSG_TYPE_ADD_REGION:
515 if ((err = memif_msg_receive_add_region (mif, &msg, fd)))
517 memif_msg_enq_ack (mif);
520 case MEMIF_MSG_TYPE_ADD_RING:
521 if ((err = memif_msg_receive_add_ring (mif, &msg, fd)))
523 memif_msg_enq_ack (mif);
526 case MEMIF_MSG_TYPE_CONNECT:
527 if ((err = memif_msg_receive_connect (mif, &msg)))
529 memif_msg_enq_connected (mif);
532 case MEMIF_MSG_TYPE_CONNECTED:
533 if ((err = memif_msg_receive_connected (mif, &msg)))
537 case MEMIF_MSG_TYPE_DISCONNECT:
538 if ((err = memif_msg_receive_disconnect (mif, &msg)))
543 err = clib_error_return (0, "unknown message type (0x%x)", msg.type);
547 if (clib_fifo_elts (mif->msg_queue) && mif->conn_unix_file_index != ~0)
548 unix_file_set_data_available_to_write (mif->conn_unix_file_index, 1);
553 memif_master_conn_fd_read_ready (unix_file_t * uf)
555 memif_main_t *mm = &memif_main;
556 memif_socket_file_t *msf =
557 pool_elt_at_index (mm->socket_files, uf->private_data);
560 uword conn_unix_file_index = ~0;
561 clib_error_t *err = 0;
563 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
566 mif = vec_elt_at_index (mm->interfaces, p[0]);
570 /* This is new connection, remove index from pending vector */
572 vec_foreach_index (i, msf->pending_file_indices)
573 if (msf->pending_file_indices[i] == uf - unix_main.file_pool)
575 conn_unix_file_index = msf->pending_file_indices[i];
576 vec_del1 (msf->pending_file_indices, i);
579 ASSERT (conn_unix_file_index != ~0);
581 err = memif_msg_receive (&mif, uf);
584 memif_disconnect (mif, err);
585 clib_error_free (err);
591 memif_slave_conn_fd_read_ready (unix_file_t * uf)
593 memif_main_t *mm = &memif_main;
595 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
596 err = memif_msg_receive (&mif, uf);
599 memif_disconnect (mif, err);
600 clib_error_free (err);
605 static clib_error_t *
606 memif_conn_fd_write_ready (unix_file_t * uf, memif_if_t * mif)
608 memif_msg_fifo_elt_t *e;
609 clib_fifo_sub2 (mif->msg_queue, e);
610 unix_file_set_data_available_to_write (mif->conn_unix_file_index, 0);
611 memif_msg_send (mif->conn_fd, &e->msg, e->fd);
616 memif_master_conn_fd_write_ready (unix_file_t * uf)
618 memif_main_t *mm = &memif_main;
619 memif_socket_file_t *msf =
620 pool_elt_at_index (mm->socket_files, uf->private_data);
624 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
628 mif = vec_elt_at_index (mm->interfaces, p[0]);
629 return memif_conn_fd_write_ready (uf, mif);
633 memif_slave_conn_fd_write_ready (unix_file_t * uf)
635 memif_main_t *mm = &memif_main;
636 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
637 return memif_conn_fd_write_ready (uf, mif);
641 memif_slave_conn_fd_error (unix_file_t * uf)
643 memif_main_t *mm = &memif_main;
644 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
647 err = clib_error_return (0, "connection fd error");
648 memif_disconnect (mif, err);
649 clib_error_free (err);
655 memif_master_conn_fd_error (unix_file_t * uf)
657 memif_main_t *mm = &memif_main;
658 memif_socket_file_t *msf =
659 pool_elt_at_index (mm->socket_files, uf->private_data);
663 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
668 mif = vec_elt_at_index (mm->interfaces, p[0]);
669 err = clib_error_return (0, "connection fd error");
670 memif_disconnect (mif, err);
671 clib_error_free (err);
676 vec_foreach_index (i, msf->pending_file_indices)
677 if (msf->pending_file_indices[i] == uf - unix_main.file_pool)
679 vec_del1 (msf->pending_file_indices, i);
685 clib_warning ("Error on unknown file descriptor %d", uf->file_descriptor);
692 memif_conn_fd_accept_ready (unix_file_t * uf)
694 memif_main_t *mm = &memif_main;
695 memif_socket_file_t *msf =
696 pool_elt_at_index (mm->socket_files, uf->private_data);
698 struct sockaddr_un client;
700 unix_file_t template = { 0 };
701 uword unix_file_index = ~0;
705 addr_len = sizeof (client);
706 conn_fd = accept (uf->file_descriptor,
707 (struct sockaddr *) &client, (socklen_t *) & addr_len);
710 return clib_error_return_unix (0, "accept fd %d", uf->file_descriptor);
712 template.read_function = memif_master_conn_fd_read_ready;
713 template.write_function = memif_master_conn_fd_write_ready;
714 template.error_function = memif_master_conn_fd_error;
715 template.file_descriptor = conn_fd;
716 template.private_data = uf->private_data;
718 memif_file_add (&unix_file_index, &template);
720 err = memif_msg_enq_hello (conn_fd);
723 clib_error_report (err);
724 memif_file_del_by_index (unix_file_index);
727 vec_add1 (msf->pending_file_indices, unix_file_index);
733 * fd.io coding-style-patch-verification: ON
736 * eval: (c-set-style "gnu")