2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/types.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
28 #include <sys/eventfd.h>
32 #include <vlib/vlib.h>
33 #include <vlib/unix/unix.h>
34 #include <vnet/plugin/plugin.h>
35 #include <vnet/ethernet/ethernet.h>
36 #include <vpp/app/version.h>
38 #include <memif/memif.h>
39 #include <memif/private.h>
42 memif_socket_close (clib_socket_t ** s)
44 memif_file_del_by_index ((*s)->private_data);
50 memif_str2vec (uint8_t * str, int len)
58 for (i = 0; i < len; i++)
70 memif_msg_enq_ack (memif_if_t * mif)
72 memif_msg_fifo_elt_t *e;
73 clib_fifo_add2 (mif->msg_queue, e);
75 e->msg.type = MEMIF_MSG_TYPE_ACK;
80 memif_msg_strlcpy (u8 * dest, u32 len, const u8 * src)
82 len = clib_min (len - 1, vec_len (src));
83 memcpy (dest, src, len);
88 memif_msg_snprintf (u8 * dest, u32 len, const char *fmt, ...)
92 u8 *s = va_format (0, fmt, &va);
94 memif_msg_strlcpy (dest, len, s);
99 memif_msg_enq_hello (clib_socket_t * sock)
101 memif_msg_t msg = { 0 };
102 memif_msg_hello_t *h = &msg.hello;
103 msg.type = MEMIF_MSG_TYPE_HELLO;
104 h->min_version = MEMIF_VERSION;
105 h->max_version = MEMIF_VERSION;
106 h->max_m2s_ring = MEMIF_MAX_M2S_RING;
107 h->max_s2m_ring = MEMIF_MAX_S2M_RING;
108 h->max_region = MEMIF_MAX_REGION;
109 h->max_log2_ring_size = MEMIF_MAX_LOG2_RING_SIZE;
110 memif_msg_snprintf (h->name, sizeof (h->name), "VPP %s", VPP_BUILD_VER);
111 return clib_socket_sendmsg (sock, &msg, sizeof (memif_msg_t), 0, 0);
115 memif_msg_enq_init (memif_if_t * mif)
117 memif_msg_fifo_elt_t *e;
118 clib_fifo_add2 (mif->msg_queue, e);
119 memif_msg_init_t *i = &e->msg.init;
121 e->msg.type = MEMIF_MSG_TYPE_INIT;
123 i->version = MEMIF_VERSION;
126 memif_msg_snprintf (i->name, sizeof (i->name), "VPP %s", VPP_BUILD_VER);
128 memif_msg_strlcpy (i->secret, sizeof (i->secret), mif->secret);
132 memif_msg_enq_add_region (memif_if_t * mif, u8 region)
134 memif_msg_fifo_elt_t *e;
135 clib_fifo_add2 (mif->msg_queue, e);
136 memif_msg_add_region_t *ar = &e->msg.add_region;
138 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
139 e->fd = mif->regions[region].fd;
141 ar->size = mif->regions[region].region_size;
145 memif_msg_enq_add_ring (memif_if_t * mif, u8 index, u8 direction)
147 memif_msg_fifo_elt_t *e;
148 clib_fifo_add2 (mif->msg_queue, e);
149 memif_msg_add_ring_t *ar = &e->msg.add_ring;
152 ASSERT ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) != 0);
154 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
156 if (direction == MEMIF_RING_M2S)
157 mq = vec_elt_at_index (mif->rx_queues, index);
159 mq = vec_elt_at_index (mif->tx_queues, index);
163 ar->region = mq->region;
164 ar->offset = mq->offset;
165 ar->log2_ring_size = mq->log2_ring_size;
166 ar->flags = (direction == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
167 ar->private_hdr_size = 0;
171 memif_msg_enq_connect (memif_if_t * mif)
173 memif_msg_fifo_elt_t *e;
174 clib_fifo_add2 (mif->msg_queue, e);
175 memif_msg_connect_t *c = &e->msg.connect;
177 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
179 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
180 format_memif_device_name, mif->dev_instance);
184 memif_msg_enq_connected (memif_if_t * mif)
186 memif_msg_fifo_elt_t *e;
187 clib_fifo_add2 (mif->msg_queue, e);
188 memif_msg_connected_t *c = &e->msg.connected;
190 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
192 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
193 format_memif_device_name, mif->dev_instance);
197 memif_msg_send_disconnect (memif_if_t * mif, clib_error_t * err)
199 memif_msg_t msg = { 0 };
200 msg.type = MEMIF_MSG_TYPE_DISCONNECT;
201 memif_msg_disconnect_t *d = &msg.disconnect;
204 memif_msg_strlcpy (d->string, sizeof (d->string), err->what);
206 return clib_socket_sendmsg (mif->sock, &msg, sizeof (memif_msg_t), 0, 0);
209 static clib_error_t *
210 memif_msg_receive_hello (memif_if_t * mif, memif_msg_t * msg)
212 memif_msg_hello_t *h = &msg->hello;
214 if (msg->hello.min_version > MEMIF_VERSION ||
215 msg->hello.max_version < MEMIF_VERSION)
216 return clib_error_return (0, "incompatible protocol version");
218 mif->run.num_s2m_rings = clib_min (h->max_s2m_ring + 1,
219 mif->cfg.num_s2m_rings);
220 mif->run.num_m2s_rings = clib_min (h->max_m2s_ring + 1,
221 mif->cfg.num_m2s_rings);
222 mif->run.log2_ring_size = clib_min (h->max_log2_ring_size,
223 mif->cfg.log2_ring_size);
224 mif->run.buffer_size = mif->cfg.buffer_size;
226 mif->remote_name = memif_str2vec (h->name, sizeof (h->name));
231 static clib_error_t *
232 memif_msg_receive_init (memif_if_t ** mifp, memif_msg_t * msg,
233 clib_socket_t * sock, uword socket_file_index)
235 memif_main_t *mm = &memif_main;
236 memif_socket_file_t *msf =
237 vec_elt_at_index (mm->socket_files, socket_file_index);
238 memif_msg_init_t *i = &msg->init;
239 memif_if_t *mif, tmp;
243 if (i->version != MEMIF_VERSION)
245 memif_file_del_by_index (sock->private_data);
246 return clib_error_return (0, "unsupported version");
249 p = mhash_get (&msf->dev_instance_by_id, &i->id);
253 err = clib_error_return (0, "unmatched interface id");
257 mif = vec_elt_at_index (mm->interfaces, p[0]);
259 if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
261 err = clib_error_return (0, "cannot connect to slave");
267 err = clib_error_return (0, "already connected");
271 if (i->mode != mif->mode)
273 err = clib_error_return (0, "mode mismatch");
278 hash_set (msf->dev_instance_by_fd, mif->sock->fd, mif->dev_instance);
279 mif->remote_name = memif_str2vec (i->name, sizeof (i->name));
286 s = memif_str2vec (i->secret, sizeof (i->secret));
288 return clib_error_return (0, "secret required");
290 r = vec_cmp (s, mif->secret);
294 return clib_error_return (0, "incorrect secret");
301 memif_msg_send_disconnect (&tmp, err);
302 memif_socket_close (&sock);
306 static clib_error_t *
307 memif_msg_receive_add_region (memif_if_t * mif, memif_msg_t * msg, int fd)
309 memif_msg_add_region_t *ar = &msg->add_region;
312 return clib_error_return (0, "missing memory region fd");
314 if (ar->index != vec_len (mif->regions))
315 return clib_error_return (0, "unexpected region index");
317 if (ar->index > MEMIF_MAX_REGION)
318 return clib_error_return (0, "too many regions");
320 vec_validate_aligned (mif->regions, ar->index, CLIB_CACHE_LINE_BYTES);
321 mr = vec_elt_at_index (mif->regions, ar->index);
323 mr->region_size = ar->size;
328 static clib_error_t *
329 memif_msg_receive_add_ring (memif_if_t * mif, memif_msg_t * msg, int fd)
331 memif_msg_add_ring_t *ar = &msg->add_ring;
335 return clib_error_return (0, "missing ring interrupt fd");
337 if (ar->private_hdr_size != 0)
338 return clib_error_return (0, "private headers not supported");
340 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M)
342 if (ar->index != vec_len (mif->rx_queues))
343 return clib_error_return (0, "unexpected ring index");
345 if (ar->index > MEMIF_MAX_S2M_RING)
346 return clib_error_return (0, "too many rings");
348 vec_validate_aligned (mif->rx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
349 mq = vec_elt_at_index (mif->rx_queues, ar->index);
350 mif->run.num_s2m_rings = vec_len (mif->rx_queues);
354 if (ar->index != vec_len (mif->tx_queues))
355 return clib_error_return (0, "unexpected ring index");
357 if (ar->index > MEMIF_MAX_M2S_RING)
358 return clib_error_return (0, "too many rings");
360 vec_validate_aligned (mif->tx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
361 mq = vec_elt_at_index (mif->tx_queues, ar->index);
362 mif->run.num_m2s_rings = vec_len (mif->tx_queues);
365 // clear previous cache data if interface reconnected
366 clib_memset (mq, 0, sizeof (memif_queue_t));
368 mq->int_clib_file_index = ~0;
369 mq->log2_ring_size = ar->log2_ring_size;
370 mq->region = ar->region;
371 mq->offset = ar->offset;
373 (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? MEMIF_RING_S2M :
379 static clib_error_t *
380 memif_msg_receive_connect (memif_if_t * mif, memif_msg_t * msg)
383 memif_msg_connect_t *c = &msg->connect;
385 if ((err = memif_connect (mif)))
388 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
393 static clib_error_t *
394 memif_msg_receive_connected (memif_if_t * mif, memif_msg_t * msg)
397 memif_msg_connected_t *c = &msg->connected;
399 if ((err = memif_connect (mif)))
402 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
406 static clib_error_t *
407 memif_msg_receive_disconnect (memif_if_t * mif, memif_msg_t * msg)
409 memif_msg_disconnect_t *d = &msg->disconnect;
411 mif->remote_disc_string = memif_str2vec (d->string, sizeof (d->string));
412 return clib_error_return (0, "disconnect received");
415 static clib_error_t *
416 memif_msg_receive (memif_if_t ** mifp, clib_socket_t * sock, clib_file_t * uf)
418 memif_msg_t msg = { 0 };
419 clib_error_t *err = 0;
422 memif_if_t *mif = *mifp;
424 err = clib_socket_recvmsg (sock, &msg, sizeof (memif_msg_t), &fd, 1);
428 if (mif == 0 && msg.type != MEMIF_MSG_TYPE_INIT)
430 memif_socket_close (&sock);
431 err = clib_error_return (0, "unexpected message received");
435 memif_log_debug (mif, "Message type %u received", msg.type);
436 /* process the message based on its type */
439 case MEMIF_MSG_TYPE_ACK:
442 case MEMIF_MSG_TYPE_HELLO:
443 if ((err = memif_msg_receive_hello (mif, &msg)))
445 if ((err = memif_init_regions_and_queues (mif)))
447 memif_msg_enq_init (mif);
449 vec_foreach_index (i, mif->regions)
450 memif_msg_enq_add_region (mif, i);
451 vec_foreach_index (i, mif->tx_queues)
452 memif_msg_enq_add_ring (mif, i, MEMIF_RING_S2M);
453 vec_foreach_index (i, mif->rx_queues)
454 memif_msg_enq_add_ring (mif, i, MEMIF_RING_M2S);
456 memif_msg_enq_connect (mif);
459 case MEMIF_MSG_TYPE_INIT:
460 if ((err = memif_msg_receive_init (mifp, &msg, sock, uf->private_data)))
463 vec_reset_length (uf->description);
464 uf->description = format (uf->description, "%U ctl",
465 format_memif_device_name, mif->dev_instance);
466 memif_msg_enq_ack (mif);
469 case MEMIF_MSG_TYPE_ADD_REGION:
470 if ((err = memif_msg_receive_add_region (mif, &msg, fd)))
472 memif_msg_enq_ack (mif);
475 case MEMIF_MSG_TYPE_ADD_RING:
476 if ((err = memif_msg_receive_add_ring (mif, &msg, fd)))
478 memif_msg_enq_ack (mif);
481 case MEMIF_MSG_TYPE_CONNECT:
482 if ((err = memif_msg_receive_connect (mif, &msg)))
484 memif_msg_enq_connected (mif);
487 case MEMIF_MSG_TYPE_CONNECTED:
488 if ((err = memif_msg_receive_connected (mif, &msg)))
492 case MEMIF_MSG_TYPE_DISCONNECT:
493 if ((err = memif_msg_receive_disconnect (mif, &msg)))
498 err = clib_error_return (0, "unknown message type (0x%x)", msg.type);
502 if (clib_fifo_elts (mif->msg_queue))
503 clib_file_set_data_available_to_write (&file_main,
504 mif->sock->private_data, 1);
508 memif_log_err (mif, "%U", format_clib_error, err);
513 memif_master_conn_fd_read_ready (clib_file_t * uf)
515 memif_main_t *mm = &memif_main;
516 memif_socket_file_t *msf =
517 pool_elt_at_index (mm->socket_files, uf->private_data);
520 clib_socket_t *sock = 0;
521 clib_error_t *err = 0;
523 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
526 mif = vec_elt_at_index (mm->interfaces, p[0]);
531 /* This is new connection, remove index from pending vector */
533 vec_foreach_index (i, msf->pending_clients)
534 if (msf->pending_clients[i]->fd == uf->file_descriptor)
536 sock = msf->pending_clients[i];
537 vec_del1 (msf->pending_clients, i);
542 err = memif_msg_receive (&mif, sock, uf);
545 memif_disconnect (mif, err);
546 clib_error_free (err);
552 memif_slave_conn_fd_read_ready (clib_file_t * uf)
554 memif_main_t *mm = &memif_main;
556 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
557 err = memif_msg_receive (&mif, mif->sock, uf);
560 memif_disconnect (mif, err);
561 clib_error_free (err);
566 static clib_error_t *
567 memif_conn_fd_write_ready (clib_file_t * uf, memif_if_t * mif)
569 memif_msg_fifo_elt_t *e;
570 clib_fifo_sub2 (mif->msg_queue, e);
571 clib_file_set_data_available_to_write (&file_main,
572 mif->sock->private_data, 0);
573 return clib_socket_sendmsg (mif->sock, &e->msg, sizeof (memif_msg_t),
574 &e->fd, e->fd > -1 ? 1 : 0);
578 memif_master_conn_fd_write_ready (clib_file_t * uf)
580 memif_main_t *mm = &memif_main;
581 memif_socket_file_t *msf =
582 pool_elt_at_index (mm->socket_files, uf->private_data);
586 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
590 mif = vec_elt_at_index (mm->interfaces, p[0]);
591 return memif_conn_fd_write_ready (uf, mif);
595 memif_slave_conn_fd_write_ready (clib_file_t * uf)
597 memif_main_t *mm = &memif_main;
598 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
599 return memif_conn_fd_write_ready (uf, mif);
603 memif_slave_conn_fd_error (clib_file_t * uf)
605 memif_main_t *mm = &memif_main;
606 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
609 err = clib_error_return (0, "connection fd error");
610 memif_disconnect (mif, err);
611 clib_error_free (err);
617 memif_master_conn_fd_error (clib_file_t * uf)
619 memif_main_t *mm = &memif_main;
620 memif_socket_file_t *msf =
621 pool_elt_at_index (mm->socket_files, uf->private_data);
625 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
630 mif = vec_elt_at_index (mm->interfaces, p[0]);
631 err = clib_error_return (0, "connection fd error");
632 memif_disconnect (mif, err);
633 clib_error_free (err);
638 vec_foreach_index (i, msf->pending_clients)
639 if (msf->pending_clients[i]->fd == uf->file_descriptor)
641 clib_socket_t *s = msf->pending_clients[i];
642 memif_socket_close (&s);
643 vec_del1 (msf->pending_clients, i);
648 memif_log_warn (0, "Error on unknown file descriptor %d",
649 uf->file_descriptor);
650 if (uf->file_descriptor != ~0)
657 memif_conn_fd_accept_ready (clib_file_t * uf)
659 memif_main_t *mm = &memif_main;
660 memif_socket_file_t *msf =
661 pool_elt_at_index (mm->socket_files, uf->private_data);
662 clib_file_t template = { 0 };
664 clib_socket_t *client;
666 client = clib_mem_alloc (sizeof (clib_socket_t));
667 clib_memset (client, 0, sizeof (clib_socket_t));
668 err = clib_socket_accept (msf->sock, client);
672 template.read_function = memif_master_conn_fd_read_ready;
673 template.write_function = memif_master_conn_fd_write_ready;
674 template.error_function = memif_master_conn_fd_error;
675 template.file_descriptor = client->fd;
676 template.private_data = uf->private_data;
677 template.description = format (0, "memif in conn on %s", msf->filename);
679 memif_file_add (&client->private_data, &template);
681 err = memif_msg_enq_hello (client);
684 clib_socket_close (client);
688 vec_add1 (msf->pending_clients, client);
693 memif_log_err (0, "%U", format_clib_error, err);
694 clib_mem_free (client);
699 * fd.io coding-style-patch-verification: ON
702 * eval: (c-set-style "gnu")