2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/types.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
28 #include <sys/eventfd.h>
32 #include <vlib/vlib.h>
33 #include <vlib/unix/unix.h>
34 #include <vnet/plugin/plugin.h>
35 #include <vnet/ethernet/ethernet.h>
36 #include <vpp/app/version.h>
38 #include <memif/memif.h>
39 #include <memif/private.h>
42 memif_socket_close (clib_socket_t ** s)
44 memif_file_del_by_index ((*s)->private_data);
50 memif_str2vec (uint8_t * str, int len)
58 for (i = 0; i < len; i++)
70 memif_msg_enq_ack (memif_if_t * mif)
72 memif_msg_fifo_elt_t *e;
73 clib_fifo_add2 (mif->msg_queue, e);
75 e->msg.type = MEMIF_MSG_TYPE_ACK;
80 memif_msg_strlcpy (u8 * dest, u32 len, const u8 * src)
82 len = clib_min (len - 1, vec_len (src));
83 memcpy (dest, src, len);
88 memif_msg_snprintf (u8 * dest, u32 len, const char *fmt, ...)
92 u8 *s = va_format (0, fmt, &va);
94 memif_msg_strlcpy (dest, len, s);
99 memif_msg_enq_hello (clib_socket_t * sock)
101 memif_msg_t msg = { 0 };
102 memif_msg_hello_t *h = &msg.hello;
103 msg.type = MEMIF_MSG_TYPE_HELLO;
104 h->min_version = MEMIF_VERSION;
105 h->max_version = MEMIF_VERSION;
106 h->max_m2s_ring = MEMIF_MAX_M2S_RING;
107 h->max_s2m_ring = MEMIF_MAX_S2M_RING;
108 h->max_region = MEMIF_MAX_REGION;
109 h->max_log2_ring_size = MEMIF_MAX_LOG2_RING_SIZE;
110 memif_msg_snprintf (h->name, sizeof (h->name), "VPP %s", VPP_BUILD_VER);
111 return clib_socket_sendmsg (sock, &msg, sizeof (memif_msg_t), 0, 0);
115 memif_msg_enq_init (memif_if_t * mif)
117 memif_msg_fifo_elt_t *e;
118 clib_fifo_add2 (mif->msg_queue, e);
119 memif_msg_init_t *i = &e->msg.init;
121 e->msg.type = MEMIF_MSG_TYPE_INIT;
123 i->version = MEMIF_VERSION;
126 memif_msg_snprintf (i->name, sizeof (i->name), "VPP %s", VPP_BUILD_VER);
128 memif_msg_strlcpy (i->secret, sizeof (i->secret), mif->secret);
132 memif_msg_enq_add_region (memif_if_t * mif, u8 region)
134 memif_msg_fifo_elt_t *e;
135 clib_fifo_add2 (mif->msg_queue, e);
136 memif_msg_add_region_t *ar = &e->msg.add_region;
138 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
139 e->fd = mif->regions[region].fd;
141 ar->size = mif->regions[region].region_size;
145 memif_msg_enq_add_ring (memif_if_t * mif, u8 index, u8 direction)
147 memif_msg_fifo_elt_t *e;
148 clib_fifo_add2 (mif->msg_queue, e);
149 memif_msg_add_ring_t *ar = &e->msg.add_ring;
152 ASSERT ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) != 0);
154 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
156 if (direction == MEMIF_RING_M2S)
157 mq = vec_elt_at_index (mif->rx_queues, index);
159 mq = vec_elt_at_index (mif->tx_queues, index);
163 ar->region = mq->region;
164 ar->offset = mq->offset;
165 ar->log2_ring_size = mq->log2_ring_size;
166 ar->flags = (direction == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
167 ar->private_hdr_size = 0;
171 memif_msg_enq_connect (memif_if_t * mif)
173 memif_msg_fifo_elt_t *e;
174 clib_fifo_add2 (mif->msg_queue, e);
175 memif_msg_connect_t *c = &e->msg.connect;
177 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
179 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
180 format_memif_device_name, mif->dev_instance);
184 memif_msg_enq_connected (memif_if_t * mif)
186 memif_msg_fifo_elt_t *e;
187 clib_fifo_add2 (mif->msg_queue, e);
188 memif_msg_connected_t *c = &e->msg.connected;
190 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
192 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
193 format_memif_device_name, mif->dev_instance);
197 memif_msg_send_disconnect (memif_if_t * mif, clib_error_t * err)
199 memif_msg_t msg = { 0 };
200 msg.type = MEMIF_MSG_TYPE_DISCONNECT;
201 memif_msg_disconnect_t *d = &msg.disconnect;
204 memif_msg_strlcpy (d->string, sizeof (d->string), err->what);
206 return clib_socket_sendmsg (mif->sock, &msg, sizeof (memif_msg_t), 0, 0);
209 static clib_error_t *
210 memif_msg_receive_hello (memif_if_t * mif, memif_msg_t * msg)
212 memif_msg_hello_t *h = &msg->hello;
214 if (msg->hello.min_version > MEMIF_VERSION ||
215 msg->hello.max_version < MEMIF_VERSION)
216 return clib_error_return (0, "incompatible protocol version");
218 mif->run.num_s2m_rings = clib_min (h->max_s2m_ring + 1,
219 mif->cfg.num_s2m_rings);
220 mif->run.num_m2s_rings = clib_min (h->max_m2s_ring + 1,
221 mif->cfg.num_m2s_rings);
222 mif->run.log2_ring_size = clib_min (h->max_log2_ring_size,
223 mif->cfg.log2_ring_size);
224 mif->run.buffer_size = mif->cfg.buffer_size;
226 mif->remote_name = memif_str2vec (h->name, sizeof (h->name));
231 static clib_error_t *
232 memif_msg_receive_init (memif_if_t ** mifp, memif_msg_t * msg,
233 clib_socket_t * sock, uword socket_file_index)
235 memif_main_t *mm = &memif_main;
236 memif_socket_file_t *msf =
237 vec_elt_at_index (mm->socket_files, socket_file_index);
238 memif_msg_init_t *i = &msg->init;
239 memif_if_t *mif, tmp;
243 if (i->version != MEMIF_VERSION)
245 memif_file_del_by_index (sock->private_data);
246 return clib_error_return (0, "unsupported version");
249 p = mhash_get (&msf->dev_instance_by_id, &i->id);
253 err = clib_error_return (0, "unmatched interface id");
257 mif = vec_elt_at_index (mm->interfaces, p[0]);
259 if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
261 err = clib_error_return (0, "cannot connect to slave");
267 err = clib_error_return (0, "already connected");
271 if (i->mode != mif->mode)
273 err = clib_error_return (0, "mode mismatch");
278 hash_set (msf->dev_instance_by_fd, mif->sock->fd, mif->dev_instance);
279 mif->remote_name = memif_str2vec (i->name, sizeof (i->name));
286 s = memif_str2vec (i->secret, sizeof (i->secret));
288 return clib_error_return (0, "secret required");
290 r = vec_cmp (s, mif->secret);
294 return clib_error_return (0, "incorrect secret");
301 memif_msg_send_disconnect (&tmp, err);
302 memif_socket_close (&sock);
306 static clib_error_t *
307 memif_msg_receive_add_region (memif_if_t * mif, memif_msg_t * msg, int fd)
309 memif_msg_add_region_t *ar = &msg->add_region;
312 return clib_error_return (0, "missing memory region fd");
314 if (ar->index != vec_len (mif->regions))
315 return clib_error_return (0, "unexpected region index");
317 if (ar->index > MEMIF_MAX_REGION)
318 return clib_error_return (0, "too many regions");
320 vec_validate_aligned (mif->regions, ar->index, CLIB_CACHE_LINE_BYTES);
321 mr = vec_elt_at_index (mif->regions, ar->index);
323 mr->region_size = ar->size;
328 static clib_error_t *
329 memif_msg_receive_add_ring (memif_if_t * mif, memif_msg_t * msg, int fd)
331 memif_msg_add_ring_t *ar = &msg->add_ring;
335 return clib_error_return (0, "missing ring interrupt fd");
337 if (ar->private_hdr_size != 0)
338 return clib_error_return (0, "private headers not supported");
340 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M)
342 if (ar->index != vec_len (mif->rx_queues))
343 return clib_error_return (0, "unexpected ring index");
345 if (ar->index > MEMIF_MAX_S2M_RING)
346 return clib_error_return (0, "too many rings");
348 vec_validate_aligned (mif->rx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
349 mq = vec_elt_at_index (mif->rx_queues, ar->index);
350 mif->run.num_s2m_rings = vec_len (mif->rx_queues);
354 if (ar->index != vec_len (mif->tx_queues))
355 return clib_error_return (0, "unexpected ring index");
357 if (ar->index > MEMIF_MAX_M2S_RING)
358 return clib_error_return (0, "too many rings");
360 vec_validate_aligned (mif->tx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
361 mq = vec_elt_at_index (mif->tx_queues, ar->index);
362 mif->run.num_m2s_rings = vec_len (mif->tx_queues);
365 // clear previous cache data if interface reconnected
366 clib_memset (mq, 0, sizeof (memif_queue_t));
368 mq->int_clib_file_index = ~0;
369 mq->log2_ring_size = ar->log2_ring_size;
370 mq->region = ar->region;
371 mq->offset = ar->offset;
373 (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? MEMIF_RING_S2M :
379 static clib_error_t *
380 memif_msg_receive_connect (memif_if_t * mif, memif_msg_t * msg)
383 memif_msg_connect_t *c = &msg->connect;
385 if ((err = memif_connect (mif)))
388 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
393 static clib_error_t *
394 memif_msg_receive_connected (memif_if_t * mif, memif_msg_t * msg)
397 memif_msg_connected_t *c = &msg->connected;
399 if ((err = memif_connect (mif)))
402 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
406 static clib_error_t *
407 memif_msg_receive_disconnect (memif_if_t * mif, memif_msg_t * msg)
409 memif_msg_disconnect_t *d = &msg->disconnect;
411 mif->remote_disc_string = memif_str2vec (d->string, sizeof (d->string));
412 return clib_error_return (0, "disconnect received");
415 static clib_error_t *
416 memif_msg_receive (memif_if_t ** mifp, clib_socket_t * sock, clib_file_t * uf)
418 memif_msg_t msg = { 0 };
419 clib_error_t *err = 0;
422 memif_if_t *mif = *mifp;
424 err = clib_socket_recvmsg (sock, &msg, sizeof (memif_msg_t), &fd, 1);
428 if (mif == 0 && msg.type != MEMIF_MSG_TYPE_INIT)
430 memif_socket_close (&sock);
431 err = clib_error_return (0, "unexpected message received");
435 memif_log_debug (mif, "Message type %u received", msg.type);
436 /* process the message based on its type */
439 case MEMIF_MSG_TYPE_ACK:
442 case MEMIF_MSG_TYPE_HELLO:
443 if ((err = memif_msg_receive_hello (mif, &msg)))
445 if ((err = memif_init_regions_and_queues (mif)))
447 memif_msg_enq_init (mif);
448 vec_foreach_index (i, mif->regions)
449 memif_msg_enq_add_region (mif, i);
450 vec_foreach_index (i, mif->tx_queues)
451 memif_msg_enq_add_ring (mif, i, MEMIF_RING_S2M);
452 vec_foreach_index (i, mif->rx_queues)
453 memif_msg_enq_add_ring (mif, i, MEMIF_RING_M2S);
454 memif_msg_enq_connect (mif);
457 case MEMIF_MSG_TYPE_INIT:
458 if ((err = memif_msg_receive_init (mifp, &msg, sock, uf->private_data)))
461 vec_reset_length (uf->description);
462 uf->description = format (uf->description, "%U ctl",
463 format_memif_device_name, mif->dev_instance);
464 memif_msg_enq_ack (mif);
467 case MEMIF_MSG_TYPE_ADD_REGION:
468 if ((err = memif_msg_receive_add_region (mif, &msg, fd)))
470 memif_msg_enq_ack (mif);
473 case MEMIF_MSG_TYPE_ADD_RING:
474 if ((err = memif_msg_receive_add_ring (mif, &msg, fd)))
476 memif_msg_enq_ack (mif);
479 case MEMIF_MSG_TYPE_CONNECT:
480 if ((err = memif_msg_receive_connect (mif, &msg)))
482 memif_msg_enq_connected (mif);
485 case MEMIF_MSG_TYPE_CONNECTED:
486 if ((err = memif_msg_receive_connected (mif, &msg)))
490 case MEMIF_MSG_TYPE_DISCONNECT:
491 if ((err = memif_msg_receive_disconnect (mif, &msg)))
496 err = clib_error_return (0, "unknown message type (0x%x)", msg.type);
500 if (clib_fifo_elts (mif->msg_queue))
501 clib_file_set_data_available_to_write (&file_main,
502 mif->sock->private_data, 1);
506 memif_log_err (mif, "%U", format_clib_error, err);
511 memif_master_conn_fd_read_ready (clib_file_t * uf)
513 memif_main_t *mm = &memif_main;
514 memif_socket_file_t *msf =
515 pool_elt_at_index (mm->socket_files, uf->private_data);
518 clib_socket_t *sock = 0;
519 clib_error_t *err = 0;
521 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
524 mif = vec_elt_at_index (mm->interfaces, p[0]);
529 /* This is new connection, remove index from pending vector */
531 vec_foreach_index (i, msf->pending_clients)
532 if (msf->pending_clients[i]->fd == uf->file_descriptor)
534 sock = msf->pending_clients[i];
535 vec_del1 (msf->pending_clients, i);
540 err = memif_msg_receive (&mif, sock, uf);
543 memif_disconnect (mif, err);
544 clib_error_free (err);
550 memif_slave_conn_fd_read_ready (clib_file_t * uf)
552 memif_main_t *mm = &memif_main;
554 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
555 err = memif_msg_receive (&mif, mif->sock, uf);
558 memif_disconnect (mif, err);
559 clib_error_free (err);
564 static clib_error_t *
565 memif_conn_fd_write_ready (clib_file_t * uf, memif_if_t * mif)
567 memif_msg_fifo_elt_t *e;
568 clib_fifo_sub2 (mif->msg_queue, e);
569 clib_file_set_data_available_to_write (&file_main,
570 mif->sock->private_data, 0);
571 return clib_socket_sendmsg (mif->sock, &e->msg, sizeof (memif_msg_t),
572 &e->fd, e->fd > -1 ? 1 : 0);
576 memif_master_conn_fd_write_ready (clib_file_t * uf)
578 memif_main_t *mm = &memif_main;
579 memif_socket_file_t *msf =
580 pool_elt_at_index (mm->socket_files, uf->private_data);
584 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
588 mif = vec_elt_at_index (mm->interfaces, p[0]);
589 return memif_conn_fd_write_ready (uf, mif);
593 memif_slave_conn_fd_write_ready (clib_file_t * uf)
595 memif_main_t *mm = &memif_main;
596 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
597 return memif_conn_fd_write_ready (uf, mif);
601 memif_slave_conn_fd_error (clib_file_t * uf)
603 memif_main_t *mm = &memif_main;
604 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
607 err = clib_error_return (0, "connection fd error");
608 memif_disconnect (mif, err);
609 clib_error_free (err);
615 memif_master_conn_fd_error (clib_file_t * uf)
617 memif_main_t *mm = &memif_main;
618 memif_socket_file_t *msf =
619 pool_elt_at_index (mm->socket_files, uf->private_data);
623 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
628 mif = vec_elt_at_index (mm->interfaces, p[0]);
629 err = clib_error_return (0, "connection fd error");
630 memif_disconnect (mif, err);
631 clib_error_free (err);
636 vec_foreach_index (i, msf->pending_clients)
637 if (msf->pending_clients[i]->fd == uf->file_descriptor)
639 clib_socket_t *s = msf->pending_clients[i];
640 memif_socket_close (&s);
641 vec_del1 (msf->pending_clients, i);
646 memif_log_warn (0, "Error on unknown file descriptor %d",
647 uf->file_descriptor);
648 if (uf->file_descriptor != ~0)
655 memif_conn_fd_accept_ready (clib_file_t * uf)
657 memif_main_t *mm = &memif_main;
658 memif_socket_file_t *msf =
659 pool_elt_at_index (mm->socket_files, uf->private_data);
660 clib_file_t template = { 0 };
662 clib_socket_t *client;
664 client = clib_mem_alloc (sizeof (clib_socket_t));
665 clib_memset (client, 0, sizeof (clib_socket_t));
666 err = clib_socket_accept (msf->sock, client);
670 template.read_function = memif_master_conn_fd_read_ready;
671 template.write_function = memif_master_conn_fd_write_ready;
672 template.error_function = memif_master_conn_fd_error;
673 template.file_descriptor = client->fd;
674 template.private_data = uf->private_data;
675 template.description = format (0, "memif in conn on %s", msf->filename);
677 memif_file_add (&client->private_data, &template);
679 err = memif_msg_enq_hello (client);
682 clib_socket_close (client);
686 vec_add1 (msf->pending_clients, client);
691 memif_log_err (0, "%U", format_clib_error, err);
692 clib_mem_free (client);
697 * fd.io coding-style-patch-verification: ON
700 * eval: (c-set-style "gnu")