2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/types.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
28 #include <sys/prctl.h>
29 #include <sys/eventfd.h>
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 #include <vnet/plugin/plugin.h>
36 #include <vnet/ethernet/ethernet.h>
37 #include <vpp/app/version.h>
39 #include <memif/memif.h>
40 #include <memif/private.h>
43 memif_socket_close (clib_socket_t ** s)
45 memif_file_del_by_index ((*s)->private_data);
51 memif_str2vec (uint8_t * str, int len)
59 for (i = 0; i < len; i++)
71 memif_msg_enq_ack (memif_if_t * mif)
73 memif_msg_fifo_elt_t *e;
74 clib_fifo_add2 (mif->msg_queue, e);
76 e->msg.type = MEMIF_MSG_TYPE_ACK;
81 memif_msg_strlcpy (u8 * dest, u32 len, const u8 * src)
83 len = clib_min (len - 1, vec_len (src));
84 memcpy (dest, src, len);
89 memif_msg_snprintf (u8 * dest, u32 len, const char *fmt, ...)
93 u8 *s = va_format (0, fmt, &va);
95 memif_msg_strlcpy (dest, len, s);
100 memif_msg_enq_hello (clib_socket_t * sock)
102 memif_msg_t msg = { 0 };
103 memif_msg_hello_t *h = &msg.hello;
104 msg.type = MEMIF_MSG_TYPE_HELLO;
105 h->min_version = MEMIF_VERSION;
106 h->max_version = MEMIF_VERSION;
107 h->max_m2s_ring = MEMIF_MAX_M2S_RING;
108 h->max_s2m_ring = MEMIF_MAX_S2M_RING;
109 h->max_region = MEMIF_MAX_REGION;
110 h->max_log2_ring_size = MEMIF_MAX_LOG2_RING_SIZE;
111 memif_msg_snprintf (h->name, sizeof (h->name), "VPP %s", VPP_BUILD_VER);
112 return clib_socket_sendmsg (sock, &msg, sizeof (memif_msg_t), 0, 0);
116 memif_msg_enq_init (memif_if_t * mif)
118 memif_msg_fifo_elt_t *e;
119 clib_fifo_add2 (mif->msg_queue, e);
120 memif_msg_init_t *i = &e->msg.init;
122 e->msg.type = MEMIF_MSG_TYPE_INIT;
124 i->version = MEMIF_VERSION;
127 memif_msg_snprintf (i->name, sizeof (i->name), "VPP %s", VPP_BUILD_VER);
129 memif_msg_strlcpy (i->secret, sizeof (i->secret), mif->secret);
133 memif_msg_enq_add_region (memif_if_t * mif, u8 region)
135 memif_msg_fifo_elt_t *e;
136 clib_fifo_add2 (mif->msg_queue, e);
137 memif_msg_add_region_t *ar = &e->msg.add_region;
139 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
140 e->fd = mif->regions[region].fd;
142 ar->size = mif->regions[region].region_size;
146 memif_msg_enq_add_ring (memif_if_t * mif, u8 index, u8 direction)
148 memif_msg_fifo_elt_t *e;
149 clib_fifo_add2 (mif->msg_queue, e);
150 memif_msg_add_ring_t *ar = &e->msg.add_ring;
153 ASSERT ((mif->flags & MEMIF_IF_FLAG_IS_SLAVE) != 0);
155 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
157 if (direction == MEMIF_RING_M2S)
158 mq = vec_elt_at_index (mif->rx_queues, index);
160 mq = vec_elt_at_index (mif->tx_queues, index);
164 ar->region = mq->region;
165 ar->offset = mq->offset;
166 ar->log2_ring_size = mq->log2_ring_size;
167 ar->flags = (direction == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
168 ar->private_hdr_size = 0;
172 memif_msg_enq_connect (memif_if_t * mif)
174 memif_msg_fifo_elt_t *e;
175 clib_fifo_add2 (mif->msg_queue, e);
176 memif_msg_connect_t *c = &e->msg.connect;
178 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
180 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
181 format_memif_device_name, mif->dev_instance);
185 memif_msg_enq_connected (memif_if_t * mif)
187 memif_msg_fifo_elt_t *e;
188 clib_fifo_add2 (mif->msg_queue, e);
189 memif_msg_connected_t *c = &e->msg.connected;
191 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
193 memif_msg_snprintf (c->if_name, sizeof (c->if_name), "%U",
194 format_memif_device_name, mif->dev_instance);
198 memif_msg_send_disconnect (memif_if_t * mif, clib_error_t * err)
200 memif_msg_t msg = { 0 };
201 msg.type = MEMIF_MSG_TYPE_DISCONNECT;
202 memif_msg_disconnect_t *d = &msg.disconnect;
205 memif_msg_strlcpy (d->string, sizeof (d->string), err->what);
207 return clib_socket_sendmsg (mif->sock, &msg, sizeof (memif_msg_t), 0, 0);
210 static clib_error_t *
211 memif_msg_receive_hello (memif_if_t * mif, memif_msg_t * msg)
213 memif_msg_hello_t *h = &msg->hello;
215 if (msg->hello.min_version > MEMIF_VERSION ||
216 msg->hello.max_version < MEMIF_VERSION)
217 return clib_error_return (0, "incompatible protocol version");
219 mif->run.num_s2m_rings = clib_min (h->max_s2m_ring + 1,
220 mif->cfg.num_s2m_rings);
221 mif->run.num_m2s_rings = clib_min (h->max_m2s_ring + 1,
222 mif->cfg.num_m2s_rings);
223 mif->run.log2_ring_size = clib_min (h->max_log2_ring_size,
224 mif->cfg.log2_ring_size);
225 mif->run.buffer_size = mif->cfg.buffer_size;
227 mif->remote_name = memif_str2vec (h->name, sizeof (h->name));
232 static clib_error_t *
233 memif_msg_receive_init (memif_if_t ** mifp, memif_msg_t * msg,
234 clib_socket_t * sock, uword socket_file_index)
236 memif_main_t *mm = &memif_main;
237 memif_socket_file_t *msf =
238 vec_elt_at_index (mm->socket_files, socket_file_index);
239 memif_msg_init_t *i = &msg->init;
240 memif_if_t *mif, tmp;
244 if (i->version != MEMIF_VERSION)
246 memif_file_del_by_index (sock->private_data);
247 return clib_error_return (0, "unsupported version");
250 p = mhash_get (&msf->dev_instance_by_id, &i->id);
254 err = clib_error_return (0, "unmatched interface id");
258 mif = vec_elt_at_index (mm->interfaces, p[0]);
260 if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
262 err = clib_error_return (0, "cannot connect to slave");
268 err = clib_error_return (0, "already connected");
272 if (i->mode != mif->mode)
274 err = clib_error_return (0, "mode mismatch");
279 hash_set (msf->dev_instance_by_fd, mif->sock->fd, mif->dev_instance);
280 mif->remote_name = memif_str2vec (i->name, sizeof (i->name));
287 s = memif_str2vec (i->secret, sizeof (i->secret));
289 return clib_error_return (0, "secret required");
291 r = vec_cmp (s, mif->secret);
295 return clib_error_return (0, "incorrect secret");
302 memif_msg_send_disconnect (&tmp, err);
303 memif_socket_close (&sock);
307 static clib_error_t *
308 memif_msg_receive_add_region (memif_if_t * mif, memif_msg_t * msg, int fd)
310 memif_msg_add_region_t *ar = &msg->add_region;
313 return clib_error_return (0, "missing memory region fd");
315 if (ar->index != vec_len (mif->regions))
316 return clib_error_return (0, "unexpected region index");
318 if (ar->index > MEMIF_MAX_REGION)
319 return clib_error_return (0, "too many regions");
321 vec_validate_aligned (mif->regions, ar->index, CLIB_CACHE_LINE_BYTES);
322 mr = vec_elt_at_index (mif->regions, ar->index);
324 mr->region_size = ar->size;
329 static clib_error_t *
330 memif_msg_receive_add_ring (memif_if_t * mif, memif_msg_t * msg, int fd)
332 memif_msg_add_ring_t *ar = &msg->add_ring;
336 return clib_error_return (0, "missing ring interrupt fd");
338 if (ar->private_hdr_size != 0)
339 return clib_error_return (0, "private headers not supported");
341 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M)
343 if (ar->index != vec_len (mif->rx_queues))
344 return clib_error_return (0, "unexpected ring index");
346 if (ar->index > MEMIF_MAX_S2M_RING)
347 return clib_error_return (0, "too many rings");
349 vec_validate_aligned (mif->rx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
350 mq = vec_elt_at_index (mif->rx_queues, ar->index);
351 mif->run.num_s2m_rings = vec_len (mif->rx_queues);
355 if (ar->index != vec_len (mif->tx_queues))
356 return clib_error_return (0, "unexpected ring index");
358 if (ar->index > MEMIF_MAX_M2S_RING)
359 return clib_error_return (0, "too many rings");
361 vec_validate_aligned (mif->tx_queues, ar->index, CLIB_CACHE_LINE_BYTES);
362 mq = vec_elt_at_index (mif->tx_queues, ar->index);
363 mif->run.num_m2s_rings = vec_len (mif->tx_queues);
366 // clear previous cache data if interface reconncected
367 clib_memset (mq, 0, sizeof (memif_queue_t));
369 mq->int_clib_file_index = ~0;
370 mq->log2_ring_size = ar->log2_ring_size;
371 mq->region = ar->region;
372 mq->offset = ar->offset;
374 (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ? MEMIF_RING_S2M :
380 static clib_error_t *
381 memif_msg_receive_connect (memif_if_t * mif, memif_msg_t * msg)
384 memif_msg_connect_t *c = &msg->connect;
386 if ((err = memif_connect (mif)))
389 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
394 static clib_error_t *
395 memif_msg_receive_connected (memif_if_t * mif, memif_msg_t * msg)
398 memif_msg_connected_t *c = &msg->connected;
400 if ((err = memif_connect (mif)))
403 mif->remote_if_name = memif_str2vec (c->if_name, sizeof (c->if_name));
407 static clib_error_t *
408 memif_msg_receive_disconnect (memif_if_t * mif, memif_msg_t * msg)
410 memif_msg_disconnect_t *d = &msg->disconnect;
412 mif->remote_disc_string = memif_str2vec (d->string, sizeof (d->string));
413 return clib_error_return (0, "disconnect received");
416 static clib_error_t *
417 memif_msg_receive (memif_if_t ** mifp, clib_socket_t * sock, clib_file_t * uf)
419 memif_msg_t msg = { 0 };
420 clib_error_t *err = 0;
423 memif_if_t *mif = *mifp;
425 err = clib_socket_recvmsg (sock, &msg, sizeof (memif_msg_t), &fd, 1);
429 if (mif == 0 && msg.type != MEMIF_MSG_TYPE_INIT)
431 memif_socket_close (&sock);
432 err = clib_error_return (0, "unexpected message received");
436 memif_log_debug (mif, "Message type %u received", msg.type);
437 /* process the message based on its type */
440 case MEMIF_MSG_TYPE_ACK:
443 case MEMIF_MSG_TYPE_HELLO:
444 if ((err = memif_msg_receive_hello (mif, &msg)))
446 if ((err = memif_init_regions_and_queues (mif)))
448 memif_msg_enq_init (mif);
450 vec_foreach_index (i, mif->regions)
451 memif_msg_enq_add_region (mif, i);
452 vec_foreach_index (i, mif->tx_queues)
453 memif_msg_enq_add_ring (mif, i, MEMIF_RING_S2M);
454 vec_foreach_index (i, mif->rx_queues)
455 memif_msg_enq_add_ring (mif, i, MEMIF_RING_M2S);
457 memif_msg_enq_connect (mif);
460 case MEMIF_MSG_TYPE_INIT:
461 if ((err = memif_msg_receive_init (mifp, &msg, sock, uf->private_data)))
464 vec_reset_length (uf->description);
465 uf->description = format (uf->description, "%U ctl",
466 format_memif_device_name, mif->dev_instance);
467 memif_msg_enq_ack (mif);
470 case MEMIF_MSG_TYPE_ADD_REGION:
471 if ((err = memif_msg_receive_add_region (mif, &msg, fd)))
473 memif_msg_enq_ack (mif);
476 case MEMIF_MSG_TYPE_ADD_RING:
477 if ((err = memif_msg_receive_add_ring (mif, &msg, fd)))
479 memif_msg_enq_ack (mif);
482 case MEMIF_MSG_TYPE_CONNECT:
483 if ((err = memif_msg_receive_connect (mif, &msg)))
485 memif_msg_enq_connected (mif);
488 case MEMIF_MSG_TYPE_CONNECTED:
489 if ((err = memif_msg_receive_connected (mif, &msg)))
493 case MEMIF_MSG_TYPE_DISCONNECT:
494 if ((err = memif_msg_receive_disconnect (mif, &msg)))
499 err = clib_error_return (0, "unknown message type (0x%x)", msg.type);
503 if (clib_fifo_elts (mif->msg_queue))
504 clib_file_set_data_available_to_write (&file_main,
505 mif->sock->private_data, 1);
509 memif_log_err (mif, "%U", format_clib_error, err);
514 memif_master_conn_fd_read_ready (clib_file_t * uf)
516 memif_main_t *mm = &memif_main;
517 memif_socket_file_t *msf =
518 pool_elt_at_index (mm->socket_files, uf->private_data);
521 clib_socket_t *sock = 0;
522 clib_error_t *err = 0;
524 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
527 mif = vec_elt_at_index (mm->interfaces, p[0]);
532 /* This is new connection, remove index from pending vector */
534 vec_foreach_index (i, msf->pending_clients)
535 if (msf->pending_clients[i]->fd == uf->file_descriptor)
537 sock = msf->pending_clients[i];
538 vec_del1 (msf->pending_clients, i);
543 err = memif_msg_receive (&mif, sock, uf);
546 memif_disconnect (mif, err);
547 clib_error_free (err);
553 memif_slave_conn_fd_read_ready (clib_file_t * uf)
555 memif_main_t *mm = &memif_main;
557 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
558 err = memif_msg_receive (&mif, mif->sock, uf);
561 memif_disconnect (mif, err);
562 clib_error_free (err);
567 static clib_error_t *
568 memif_conn_fd_write_ready (clib_file_t * uf, memif_if_t * mif)
570 memif_msg_fifo_elt_t *e;
571 clib_fifo_sub2 (mif->msg_queue, e);
572 clib_file_set_data_available_to_write (&file_main,
573 mif->sock->private_data, 0);
574 return clib_socket_sendmsg (mif->sock, &e->msg, sizeof (memif_msg_t),
575 &e->fd, e->fd > -1 ? 1 : 0);
579 memif_master_conn_fd_write_ready (clib_file_t * uf)
581 memif_main_t *mm = &memif_main;
582 memif_socket_file_t *msf =
583 pool_elt_at_index (mm->socket_files, uf->private_data);
587 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
591 mif = vec_elt_at_index (mm->interfaces, p[0]);
592 return memif_conn_fd_write_ready (uf, mif);
596 memif_slave_conn_fd_write_ready (clib_file_t * uf)
598 memif_main_t *mm = &memif_main;
599 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
600 return memif_conn_fd_write_ready (uf, mif);
604 memif_slave_conn_fd_error (clib_file_t * uf)
606 memif_main_t *mm = &memif_main;
607 memif_if_t *mif = vec_elt_at_index (mm->interfaces, uf->private_data);
610 err = clib_error_return (0, "connection fd error");
611 memif_disconnect (mif, err);
612 clib_error_free (err);
618 memif_master_conn_fd_error (clib_file_t * uf)
620 memif_main_t *mm = &memif_main;
621 memif_socket_file_t *msf =
622 pool_elt_at_index (mm->socket_files, uf->private_data);
626 p = hash_get (msf->dev_instance_by_fd, uf->file_descriptor);
631 mif = vec_elt_at_index (mm->interfaces, p[0]);
632 err = clib_error_return (0, "connection fd error");
633 memif_disconnect (mif, err);
634 clib_error_free (err);
639 vec_foreach_index (i, msf->pending_clients)
640 if (msf->pending_clients[i]->fd == uf->file_descriptor)
642 clib_socket_t *s = msf->pending_clients[i];
643 memif_socket_close (&s);
644 vec_del1 (msf->pending_clients, i);
649 memif_log_warn (0, "Error on unknown file descriptor %d",
650 uf->file_descriptor);
657 memif_conn_fd_accept_ready (clib_file_t * uf)
659 memif_main_t *mm = &memif_main;
660 memif_socket_file_t *msf =
661 pool_elt_at_index (mm->socket_files, uf->private_data);
662 clib_file_t template = { 0 };
664 clib_socket_t *client;
666 client = clib_mem_alloc (sizeof (clib_socket_t));
667 clib_memset (client, 0, sizeof (clib_socket_t));
668 err = clib_socket_accept (msf->sock, client);
672 template.read_function = memif_master_conn_fd_read_ready;
673 template.write_function = memif_master_conn_fd_write_ready;
674 template.error_function = memif_master_conn_fd_error;
675 template.file_descriptor = client->fd;
676 template.private_data = uf->private_data;
677 template.description = format (0, "memif in conn on %s", msf->filename);
679 memif_file_add (&client->private_data, &template);
681 err = memif_msg_enq_hello (client);
684 clib_socket_close (client);
688 vec_add1 (msf->pending_clients, client);
693 memif_log_err (0, "%U", format_clib_error, err);
694 clib_mem_free (client);
699 * fd.io coding-style-patch-verification: ON
702 * eval: (c-set-style "gnu")