2 *------------------------------------------------------------------
3 * Copyright (c) 2017 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/types.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
28 #include <sys/prctl.h>
34 #include <linux/icmp.h>
35 #include <arpa/inet.h>
37 #include <netinet/if_ether.h>
38 #include <net/if_arp.h>
39 #include <asm/byteorder.h>
44 #include <sys/eventfd.h>
45 #include <sys/timerfd.h>
46 #include <sys/epoll.h>
48 #include <linux/memfd.h>
50 /* memif protocol msg, ring and descriptor definitions */
54 /* socket messaging functions */
56 /* private structs and functions */
57 #include <memif_private.h>
59 #define ERRLIST_LEN 40
60 #define MAX_ERRBUF_LEN 256
63 #define MEMIF_MEMORY_BARRIER() __builtin_ia32_sfence ()
65 #define MEMIF_MEMORY_BARRIER() __sync_synchronize ()
66 #endif /* __x86_x64__ */
68 static char memif_buf[MAX_ERRBUF_LEN];
70 const char *memif_errlist[ERRLIST_LEN] = { /* MEMIF_ERR_SUCCESS */
72 /* MEMIF_ERR_SYSCALL */
73 "Unspecified syscall error (build with -DMEMIF_DBG or make debug).",
74 /* MEMIF_ERR_CONNREFUSED */
77 "Permission to resource denied.",
78 /* MEMIF_ERR_NO_FILE */
79 "Socket file does not exist",
80 /* MEMIF_ERR_FILE_LIMIT */
81 "System limit on total numer of open files reached.",
82 /* MEMIF_ERR_PROC_FILE_LIMIT */
83 "Per-process limit on total number of open files reached.",
84 /* MEMIF_ERR_ALREADY */
85 "Connection already requested.",
87 "File descriptor refers to file other than socket, or operation would block.",
88 /* MEMIF_ERR_BAD_FD */
89 "Bad file descriptor.",
92 /* MEMIF_ERR_INVAL_ARG */
94 /* MEMIF_ERR_NOCONN */
95 "Memif connection handle does not point to existing connection",
97 "Memif connection handle points to existing connection",
98 /* MEMIF_ERR_CB_FDUPDATE */
99 "Callback memif_control_fd_update_t returned error",
100 /* MEMIF_ERR_FILE_NOT_SOCK */
101 "File specified by socket filename exists and is not socket.",
102 /* MEMIF_ERR_NO_SHMFD */
103 "Missing shared memory file descriptor. (internal error)",
104 /* MEMIF_ERR_COOKIE */
105 "Invalid cookie on ring. (internal error)",
106 /* MEMIF_ERR_NOBUF_RING */
108 /* MEMIF_ERR_NOBUF */
109 "Not enough memif buffers. There are unreceived data in shared memory.",
110 /* MEMIF_ERR_NOBUF_DET */
111 "Not enough space for memif details in supplied buffer. String data might be malformed.",
112 /* MEMIF_ERR_INT_WRITE */
113 "Send interrupt error.",
114 /* MEMIF_ERR_MFMSG */
115 "Malformed message received on control channel.",
118 /* MEMIF_ERR_PROTO */
119 "Incompatible memory interface protocol version.",
121 "Unmatched interface id.",
122 /* MEMIF_ERR_ACCSLAVE */
123 "Slave cannot accept connection request.",
124 /* MEMIF_ERR_ALRCONN */
125 "Interface is already connected.",
128 /* MEMIF_ERR_SECRET */
130 /* MEMIF_ERR_NOSECRET */
132 /* MEMIF_ERR_MAXREG */
133 "Limit on total number of regions reached.",
134 /* MEMIF_ERR_MAXRING */
135 "Limit on total number of ring reached.",
136 /* MEMIF_ERR_NO_INTFD */
137 "Missing interrupt file descriptor. (internal error)",
138 /* MEMIF_ERR_DISCONNECT */
139 "Interface received disconnect request.",
140 /* MEMIF_ERR_DISCONNECTED */
141 "Interface is disconnected.",
142 /* MEMIF_ERR_UNKNOWN_MSG */
143 "Unknown message type received on control channel. (internal error)",
144 /* MEMIF_ERR_POLL_CANCEL */
145 "Memif event polling was canceled.",
146 /* MEMIF_ERR_MAX_RING */
147 "Maximum log2 ring size is 15",
148 /* MEMIF_ERR_PRIVHDR */
149 "Private headers not supported."
152 #define MEMIF_ERR_UNDEFINED "undefined error"
155 memif_strerror (int err_code)
157 if (err_code >= ERRLIST_LEN)
159 strlcpy (memif_buf, MEMIF_ERR_UNDEFINED, sizeof (memif_buf));
163 strlcpy (memif_buf, memif_errlist[err_code], sizeof (memif_buf));
164 memif_buf[strlen (memif_errlist[err_code])] = '\0';
172 return MEMIF_VERSION;
176 memif_get_version_str ()
178 #define __STR_HELPER(x) #x
179 #define __STR(x) __STR_HELPER (x)
180 return __STR (MEMIF_VERSION_MAJOR) "." __STR (MEMIF_VERSION_MINOR);
185 #define DBG_TX_BUF (0)
186 #define DBG_RX_BUF (1)
190 print_bytes (void *data, uint16_t len, uint8_t q)
193 printf ("\nTX:\n\t");
195 printf ("\nRX:\n\t");
197 for (i = 0; i < len; i++)
200 printf ("\n%d:\t", i);
201 printf ("%02X ", ((uint8_t *) (data))[i]);
205 #endif /* MEMIF_DBG_SHM */
208 memif_syscall_error_handler (int err_code)
210 DBG ("%s", strerror (err_code));
213 return MEMIF_ERR_SUCCESS;
214 if (err_code == EACCES)
215 return MEMIF_ERR_ACCES;
216 if (err_code == ENFILE)
217 return MEMIF_ERR_FILE_LIMIT;
218 if (err_code == EMFILE)
219 return MEMIF_ERR_PROC_FILE_LIMIT;
220 if (err_code == ENOMEM)
221 return MEMIF_ERR_NOMEM;
222 /* connection refused if master does not exist
223 this error would spam the user until master was created */
225 if (err_code == ECONNREFUSED)
226 return MEMIF_ERR_SUCCESS;
228 if (err_code == ECONNREFUSED)
229 return MEMIF_ERR_CONNREFUSED;
230 if (err_code == EALREADY)
231 return MEMIF_ERR_ALREADY;
232 if (err_code == EAGAIN)
233 return MEMIF_ERR_AGAIN;
234 if (err_code == EBADF)
235 return MEMIF_ERR_BAD_FD;
236 if (err_code == ENOENT)
237 return MEMIF_ERR_NO_FILE;
239 /* other syscall errors */
240 return MEMIF_ERR_SYSCALL;
244 memif_add_epoll_fd (memif_socket_t *ms, memif_fd_event_t fde, uint32_t events)
248 DBG ("invalid fd %d", fde.fd);
251 struct epoll_event evt;
252 memset (&evt, 0, sizeof (evt));
254 evt.data.ptr = fde.private_ctx;
255 if (epoll_ctl (ms->epfd, EPOLL_CTL_ADD, fde.fd, &evt) < 0)
257 DBG ("epoll_ctl: %s fd %d", strerror (errno), fde.fd);
260 DBG ("fd %d added to epoll", fde.fd);
265 memif_mod_epoll_fd (memif_socket_t *ms, memif_fd_event_t fde, uint32_t events)
269 DBG ("invalid fd %d", fde.fd);
272 struct epoll_event evt;
273 memset (&evt, 0, sizeof (evt));
275 evt.data.ptr = fde.private_ctx;
276 if (epoll_ctl (ms->epfd, EPOLL_CTL_MOD, fde.fd, &evt) < 0)
278 DBG ("epoll_ctl: %s fd %d", strerror (errno), fde.fd);
281 DBG ("fd %d modified on epoll", fde.fd);
286 memif_del_epoll_fd (memif_socket_t *ms, memif_fd_event_t fde)
290 DBG ("invalid fd %d", fde.fd);
293 struct epoll_event evt;
294 memset (&evt, 0, sizeof (evt));
295 if (epoll_ctl (ms->epfd, EPOLL_CTL_DEL, fde.fd, &evt) < 0)
297 DBG ("epoll_ctl: %s fd %d", strerror (errno), fde.fd);
300 DBG ("fd %d removed from epoll", fde.fd);
305 memif_control_fd_update (memif_fd_event_t fde, void *private_ctx)
307 memif_socket_t *ms = (memif_socket_t *) private_ctx;
311 return MEMIF_ERR_INVAL_ARG;
313 if (fde.type & MEMIF_FD_EVENT_DEL)
314 return memif_del_epoll_fd (ms, fde);
317 if (fde.type & MEMIF_FD_EVENT_READ)
319 if (fde.type & MEMIF_FD_EVENT_WRITE)
322 if (fde.type & MEMIF_FD_EVENT_MOD)
323 return memif_mod_epoll_fd (ms, fde, evt);
325 return memif_add_epoll_fd (ms, fde, evt);
329 memif_control_fd_update_register (memif_socket_t *ms,
330 memif_control_fd_update_t *cb)
332 ms->args.on_control_fd_update = cb;
336 memif_register_external_region (memif_socket_handle_t sock,
337 memif_add_external_region_t *ar,
338 memif_get_external_region_addr_t *gr,
339 memif_del_external_region_t *dr,
340 memif_get_external_buffer_offset_t *go)
342 memif_socket_t *ms = (memif_socket_t *) sock;
343 ms->add_external_region = ar;
344 ms->get_external_region_addr = gr;
345 ms->del_external_region = dr;
346 ms->get_external_buffer_offset = go;
350 memif_alloc_register (memif_socket_t *ms, memif_alloc_t *ma)
356 memif_realloc_register (memif_socket_t *ms, memif_realloc_t *mr)
358 ms->args.realloc = mr;
362 memif_free_register (memif_socket_t *ms, memif_free_t *mf)
367 static inline memif_ring_t *
368 memif_get_ring (memif_connection_t * conn, memif_ring_type_t type,
371 if (&conn->regions[0] == NULL)
373 void *p = conn->regions[0].addr;
375 sizeof (memif_ring_t) +
376 sizeof (memif_desc_t) * (1 << conn->run_args.log2_ring_size);
377 p += (ring_num + type * conn->run_args.num_s2m_rings) * ring_size;
379 return (memif_ring_t *) p;
383 memif_set_rx_mode (memif_conn_handle_t c, memif_rx_mode_t rx_mode,
386 memif_connection_t *conn = (memif_connection_t *) c;
388 return MEMIF_ERR_NOCONN;
390 (conn->args.is_master) ? conn->run_args.num_s2m_rings : conn->
391 run_args.num_m2s_rings;
393 return MEMIF_ERR_QID;
395 conn->rx_queues[qid].ring->flags = rx_mode;
396 DBG ("rx_mode flag: %u", conn->rx_queues[qid].ring->flags);
397 return MEMIF_ERR_SUCCESS;
401 memif_poll_cancel_handler (memif_fd_event_type_t type, void *private_ctx)
403 return MEMIF_ERR_POLL_CANCEL;
407 memif_connect_handler (memif_fd_event_type_t type, void *private_ctx)
409 memif_socket_t *ms = (memif_socket_t *) private_ctx;
410 memif_connection_t *c;
412 if (ms->timer_fd >= 0)
416 Have to read the timer fd else it stays read-ready
417 and makes epoll_pwait() return without sleeping
419 read (ms->timer_fd, &u64, sizeof (u64));
422 /* loop ms->slave_interfaces and request connection for disconnected ones */
423 TAILQ_FOREACH (c, &ms->slave_interfaces, next)
425 /* connected or connecting */
426 if (c->control_channel != NULL)
430 memif_request_connection (c);
433 return MEMIF_ERR_SUCCESS;
437 memif_set_connection_request_timer (memif_socket_handle_t sock,
438 struct itimerspec timer)
440 memif_socket_t *ms = (memif_socket_t *) sock;
441 memif_fd_event_t fde;
442 memif_fd_event_data_t *fdata;
446 return MEMIF_ERR_INVAL_ARG;
448 if (ms->timer_fd < 0)
450 /* only create new timer if there is a valid interval */
451 if (timer.it_interval.tv_sec == 0 && timer.it_interval.tv_nsec == 0)
452 return MEMIF_ERR_SUCCESS;
455 ms->timer_fd = timerfd_create (CLOCK_REALTIME, TFD_NONBLOCK);
456 if (ms->timer_fd < 0)
457 return memif_syscall_error_handler (errno);
459 /* start listening for events */
460 fdata = ms->args.alloc (sizeof (*fdata));
461 fdata->event_handler = memif_connect_handler;
462 fdata->private_ctx = ms;
464 fde.fd = ms->timer_fd;
465 fde.type = MEMIF_FD_EVENT_READ;
466 fde.private_ctx = fdata;
468 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
469 ms->args.on_control_fd_update (fde, ctx);
472 ms->args.connection_request_timer = timer;
475 if (timerfd_settime (ms->timer_fd, 0, &ms->args.connection_request_timer,
477 return memif_syscall_error_handler (errno);
479 return MEMIF_ERR_SUCCESS;
483 memif_create_socket (memif_socket_handle_t *sock, memif_socket_args_t *args,
486 memif_socket_t *ms = (memif_socket_t *) * sock;
487 memif_fd_event_t fde;
488 memif_fd_event_data_t *fdata;
489 int i, err = MEMIF_ERR_SUCCESS;
492 /* allocate memif_socket_t */
494 if (args->alloc != NULL)
495 ms = args->alloc (sizeof (memif_socket_t));
497 ms = malloc (sizeof (memif_socket_t));
500 err = MEMIF_ERR_NOMEM;
505 memset (ms, 0, sizeof (memif_socket_t));
507 ms->listener_fd = -1;
508 ms->poll_cancel_fd = -1;
511 /* copy arguments to internal struct */
512 memcpy (&ms->args, args, sizeof (*args));
513 ms->private_ctx = private_ctx;
515 if (ms->args.alloc == NULL)
516 memif_alloc_register (ms, malloc);
517 if (ms->args.realloc == NULL)
518 memif_realloc_register (ms, realloc);
519 if (ms->args.free == NULL)
520 memif_free_register (ms, free);
522 TAILQ_INIT (&ms->master_interfaces);
523 TAILQ_INIT (&ms->slave_interfaces);
525 /* FIXME: implement connection request timer */
527 /* initialize internal epoll */
528 if (ms->args.on_control_fd_update == NULL)
530 ms->epfd = epoll_create (1);
531 /* register default fd update callback */
532 memif_control_fd_update_register (ms, memif_control_fd_update);
533 ms->poll_cancel_fd = eventfd (0, EFD_NONBLOCK);
534 if (ms->poll_cancel_fd < 0)
537 DBG ("eventfd: %s", strerror (err));
538 return memif_syscall_error_handler (err);
540 /* add interrupt fd to epfd */
541 fdata = ms->args.alloc (sizeof (*fdata));
542 fdata->event_handler = memif_poll_cancel_handler;
543 fdata->private_ctx = ms;
545 fde.fd = ms->poll_cancel_fd;
546 fde.type = MEMIF_FD_EVENT_READ;
547 fde.private_ctx = fdata;
549 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
550 ms->args.on_control_fd_update (fde, ctx);
554 memif_set_connection_request_timer (ms, ms->args.connection_request_timer);
555 if (err != MEMIF_ERR_SUCCESS)
568 if (ms->poll_cancel_fd != -1)
569 close (ms->poll_cancel_fd);
574 memif_socket_handle_t
575 memif_get_socket_handle (memif_conn_handle_t conn)
577 memif_connection_t *c = (memif_connection_t *) conn;
582 return c->args.socket;
586 memif_get_socket_path (memif_socket_handle_t sock)
588 memif_socket_t *ms = (memif_socket_t *) sock;
593 return ms->args.path;
597 memif_get_listener_fd (memif_socket_handle_t sock)
599 memif_socket_t *ms = (memif_socket_t *) sock;
604 return ms->listener_fd;
608 memif_set_listener_fd (memif_socket_handle_t sock, int fd)
610 memif_socket_t *ms = (memif_socket_t *) sock;
611 memif_fd_event_t fde;
612 memif_fd_event_data_t *fdata;
615 if ((ms == NULL) || (fd < 0))
616 return MEMIF_ERR_INVAL_ARG;
618 fdata = ms->args.alloc (sizeof (*fdata));
620 return MEMIF_ERR_NOMEM;
622 ms->listener_fd = fd;
624 fdata->event_handler = memif_listener_handler;
625 fdata->private_ctx = ms;
626 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
627 /* send fd to epoll */
628 fde.fd = ms->listener_fd;
629 fde.type = MEMIF_FD_EVENT_READ;
630 fde.private_ctx = fdata;
631 ms->args.on_control_fd_update (fde, ctx);
633 return MEMIF_ERR_SUCCESS;
637 memif_create (memif_conn_handle_t *c, memif_conn_args_t *args,
638 memif_connection_update_t *on_connect,
639 memif_connection_update_t *on_disconnect,
640 memif_on_interrupt_t *on_interrupt, void *private_ctx)
643 memif_connection_t *conn = (memif_connection_t *) * c;
644 memif_socket_t *ms = (memif_socket_t *) args->socket;
648 DBG ("This handle already points to existing memif.");
649 return MEMIF_ERR_CONN;
654 DBG ("Missing memif socket");
655 return MEMIF_ERR_INVAL_ARG;
658 conn = (memif_connection_t *) ms->args.alloc (sizeof (*conn));
661 err = MEMIF_ERR_NOMEM;
664 memset (conn, 0, sizeof (memif_connection_t));
666 conn->args.interface_id = args->interface_id;
668 if (args->log2_ring_size == 0)
669 args->log2_ring_size = MEMIF_DEFAULT_LOG2_RING_SIZE;
670 else if (args->log2_ring_size > MEMIF_MAX_LOG2_RING_SIZE)
672 err = MEMIF_ERR_MAX_RING;
675 if (args->buffer_size == 0)
676 args->buffer_size = MEMIF_DEFAULT_BUFFER_SIZE;
677 if (args->num_s2m_rings == 0)
678 args->num_s2m_rings = MEMIF_DEFAULT_TX_QUEUES;
679 if (args->num_m2s_rings == 0)
680 args->num_m2s_rings = MEMIF_DEFAULT_RX_QUEUES;
682 conn->args.num_s2m_rings = args->num_s2m_rings;
683 conn->args.num_m2s_rings = args->num_m2s_rings;
684 conn->args.buffer_size = args->buffer_size;
685 conn->args.log2_ring_size = args->log2_ring_size;
686 conn->args.is_master = args->is_master;
687 conn->args.mode = args->mode;
688 conn->args.socket = args->socket;
689 conn->regions = NULL;
690 conn->tx_queues = NULL;
691 conn->rx_queues = NULL;
692 conn->control_channel = NULL;
693 conn->on_connect = on_connect;
694 conn->on_disconnect = on_disconnect;
695 conn->on_interrupt = on_interrupt;
696 conn->private_ctx = private_ctx;
697 memset (&conn->run_args, 0, sizeof (memif_conn_run_args_t));
699 uint8_t l = sizeof (conn->args.interface_name);
700 strlcpy ((char *) conn->args.interface_name, (char *) args->interface_name,
703 if ((l = strlen ((char *) args->secret)) > 0)
704 strlcpy ((char *) conn->args.secret, (char *) args->secret,
705 sizeof (conn->args.secret));
708 TAILQ_INSERT_TAIL (&ms->master_interfaces, conn, next);
710 TAILQ_INSERT_TAIL (&ms->slave_interfaces, conn, next);
712 err = memif_request_connection (conn);
713 if (err != MEMIF_ERR_SUCCESS && err != MEMIF_ERR_CONNREFUSED)
716 TAILQ_REMOVE (&ms->master_interfaces, conn, next);
718 TAILQ_REMOVE (&ms->slave_interfaces, conn, next);
728 ms->args.free (conn);
734 memif_path_is_abstract (const char *filename)
736 return (filename[0] == '@');
740 memif_request_connection (memif_conn_handle_t c)
742 memif_connection_t *conn = (memif_connection_t *) c;
744 int err = MEMIF_ERR_SUCCESS;
746 struct sockaddr_un un = { 0 };
747 struct stat file_stat;
749 memif_control_channel_t *cc = NULL;
750 memif_fd_event_t fde;
751 memif_fd_event_data_t *fdata = NULL;
752 int sunlen = sizeof (un);
756 return MEMIF_ERR_NOCONN;
758 ms = (memif_socket_t *) conn->args.socket;
760 /* if control channel is assigned, the interface is either connected or
762 if (conn->control_channel != NULL)
763 return MEMIF_ERR_ALRCONN;
764 /* if interface is master and the socket is already listener we are done */
765 if (conn->args.is_master && (ms->listener_fd != -1))
766 return MEMIF_ERR_SUCCESS;
768 sockfd = socket (AF_UNIX, SOCK_SEQPACKET, 0);
771 err = memif_syscall_error_handler (errno);
775 un.sun_family = AF_UNIX;
777 /* use memcpy to support abstract socket
778 * ms->args.path is already a valid socket path
780 memcpy (un.sun_path, ms->args.path, sizeof (un.sun_path) - 1);
782 /* allocate fd event data */
783 fdata = ms->args.alloc (sizeof (*fdata));
786 err = MEMIF_ERR_NOMEM;
790 if (memif_path_is_abstract (ms->args.path))
792 /* Ensure the string is NULL terminated */
793 un.sun_path[sizeof (un.sun_path) - 1] = '\0';
794 /* sunlen is strlen(un.sun_path) + sizeof(un.sun_family) */
795 sunlen = strlen (un.sun_path) + (sizeof (un) - sizeof (un.sun_path));
796 /* Handle abstract socket by converting '@' -> '\0' */
797 un.sun_path[0] = '\0';
800 if (conn->args.is_master != 0)
802 /* Configure socket optins */
803 if (setsockopt (sockfd, SOL_SOCKET, SO_PASSCRED, &on, sizeof (on)) < 0)
805 err = memif_syscall_error_handler (errno);
808 if (bind (sockfd, (struct sockaddr *) &un, sunlen) < 0)
810 err = memif_syscall_error_handler (errno);
813 if (listen (sockfd, 1) < 0)
815 err = memif_syscall_error_handler (errno);
818 if (!memif_path_is_abstract (ms->args.path))
820 /* Verify that the socket was created */
821 if (stat ((char *) ms->args.path, &file_stat) < 0)
823 err = memif_syscall_error_handler (errno);
828 /* assign listener fd */
829 ms->listener_fd = sockfd;
831 fdata->event_handler = memif_listener_handler;
832 fdata->private_ctx = ms;
836 cc = ms->args.alloc (sizeof (*cc));
839 err = MEMIF_ERR_NOMEM;
842 if (connect (sockfd, (struct sockaddr *) &un, sunlen) != 0)
844 err = MEMIF_ERR_CONNREFUSED;
848 /* Create control channel */
852 TAILQ_INIT (&cc->msg_queue);
854 /* assign control channel to endpoint */
855 conn->control_channel = cc;
857 fdata->event_handler = memif_control_channel_handler;
858 fdata->private_ctx = cc;
861 /* if event polling is done internally, send memif socket as context */
862 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
863 /* send fd to epoll */
865 fde.type = MEMIF_FD_EVENT_READ;
866 fde.private_ctx = fdata;
867 ms->args.on_control_fd_update (fde, ctx);
876 ms->args.free (fdata);
880 conn->control_channel = cc = NULL;
885 memif_control_fd_handler (void *ptr, memif_fd_event_type_t events)
887 memif_fd_event_data_t *fdata = (memif_fd_event_data_t *) ptr;
890 return MEMIF_ERR_INVAL_ARG;
892 return fdata->event_handler (events, fdata->private_ctx);
896 memif_interrupt_handler (memif_fd_event_type_t type, void *private_ctx)
898 memif_interrupt_t *idata = (memif_interrupt_t *) private_ctx;
901 return MEMIF_ERR_INVAL_ARG;
903 return idata->c->on_interrupt (idata->c, idata->c->private_ctx, idata->qid);
907 memif_poll_event (memif_socket_handle_t sock, int timeout)
909 memif_socket_t *ms = (memif_socket_t *) sock;
910 struct epoll_event evt;
911 int en = 0, err = MEMIF_ERR_SUCCESS; /* 0 */
912 memif_fd_event_type_t events = 0;
913 uint64_t counter = 0;
918 return MEMIF_ERR_INVAL_ARG;
920 memset (&evt, 0, sizeof (evt));
921 evt.events = EPOLLIN | EPOLLOUT;
922 sigemptyset (&sigset);
923 en = epoll_pwait (ms->epfd, &evt, 1, timeout, &sigset);
927 DBG ("epoll_pwait: %s", strerror (err));
928 return memif_syscall_error_handler (err);
932 if (evt.events & EPOLLIN)
933 events |= MEMIF_FD_EVENT_READ;
934 if (evt.events & EPOLLOUT)
935 events |= MEMIF_FD_EVENT_WRITE;
936 if (evt.events & EPOLLERR)
937 events |= MEMIF_FD_EVENT_ERROR;
938 return memif_control_fd_handler (evt.data.ptr, events);
940 return MEMIF_ERR_SUCCESS;
944 memif_cancel_poll_event (memif_socket_handle_t sock)
946 memif_socket_t *ms = (memif_socket_t *) sock;
947 uint64_t counter = 1;
950 if (ms->poll_cancel_fd == -1)
951 return MEMIF_ERR_INVAL_ARG;
952 w = write (ms->poll_cancel_fd, &counter, sizeof (counter));
953 if (w < sizeof (counter))
954 return MEMIF_ERR_INT_WRITE;
956 return MEMIF_ERR_SUCCESS;
960 memif_close_queues (memif_socket_t *ms, memif_queue_t *queues, int nqueues)
962 memif_fd_event_t fde;
967 for (i = 0; i < nqueues; i++)
974 /* Stop listening for events */
976 fde.type = MEMIF_FD_EVENT_DEL;
977 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
978 ms->args.on_control_fd_update (fde, ctx);
986 /* send disconnect msg and close interface */
988 memif_disconnect_internal (memif_connection_t * c)
990 int err = MEMIF_ERR_SUCCESS, i; /* 0 */
992 memif_socket_t *ms = (memif_socket_t *) c->args.socket;
993 memif_fd_event_t fde;
996 c->on_disconnect ((void *) c, c->private_ctx);
998 /* Delete control channel */
999 if (c->control_channel != NULL)
1000 memif_delete_control_channel (c->control_channel);
1002 if (c->tx_queues != NULL)
1004 memif_close_queues (ms, c->tx_queues, c->tx_queues_num);
1005 ms->args.free (c->tx_queues);
1006 c->tx_queues = NULL;
1008 c->tx_queues_num = 0;
1010 if (c->rx_queues != NULL)
1012 memif_close_queues (ms, c->rx_queues, c->rx_queues_num);
1013 ms->args.free (c->rx_queues);
1014 c->rx_queues = NULL;
1016 c->rx_queues_num = 0;
1018 /* TODO: Slave reuse regions */
1020 for (i = 0; i < c->regions_num; i++)
1022 if (&c->regions[i] == NULL)
1024 if (c->regions[i].is_external != 0)
1026 ms->del_external_region (c->regions[i].addr,
1027 c->regions[i].region_size, c->regions[i].fd,
1032 if (munmap (c->regions[i].addr, c->regions[i].region_size) < 0)
1033 return memif_syscall_error_handler (errno);
1034 if (c->regions[i].fd > 0)
1035 close (c->regions[i].fd);
1036 c->regions[i].fd = -1;
1039 ms->args.free (c->regions);
1043 memset (&c->run_args, 0, sizeof (memif_conn_run_args_t));
1049 memif_get_socket_filename (memif_socket_handle_t sock)
1051 memif_socket_t *ms = (memif_socket_t *) sock;
1056 return (char *) ms->args.path;
1060 memif_delete_socket (memif_socket_handle_t * sock)
1062 memif_socket_t *ms = (memif_socket_t *) * sock;
1063 memif_fd_event_t fde;
1066 /* check if socket is in use */
1067 if (ms == NULL || !TAILQ_EMPTY (&ms->master_interfaces) ||
1068 !TAILQ_EMPTY (&ms->slave_interfaces))
1069 return MEMIF_ERR_INVAL_ARG;
1071 if (ms->listener_fd > 0)
1073 fde.fd = ms->listener_fd;
1074 fde.type = MEMIF_FD_EVENT_DEL;
1075 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
1076 ms->args.on_control_fd_update (fde, ctx);
1078 ms->listener_fd = -1;
1080 if (ms->poll_cancel_fd > 0)
1082 fde.fd = ms->poll_cancel_fd;
1083 fde.type = MEMIF_FD_EVENT_DEL;
1084 ctx = ms->epfd != -1 ? ms : ms->private_ctx;
1085 ms->args.on_control_fd_update (fde, ctx);
1087 ms->poll_cancel_fd = -1;
1096 return MEMIF_ERR_SUCCESS;
1100 memif_delete (memif_conn_handle_t * conn)
1102 memif_connection_t *c = (memif_connection_t *) * conn;
1104 int err = MEMIF_ERR_SUCCESS;
1108 DBG ("no connection");
1109 return MEMIF_ERR_NOCONN;
1112 err = memif_disconnect_internal (c);
1114 ms = (memif_socket_t *) c->args.socket;
1116 if (c->args.is_master)
1117 TAILQ_REMOVE (&ms->master_interfaces, c, next);
1119 TAILQ_REMOVE (&ms->slave_interfaces, c, next);
1120 /* TODO: don't listen with empty interface queue */
1130 memif_connect1 (memif_connection_t * c)
1138 return MEMIF_ERR_INVAL_ARG;
1140 ms = (memif_socket_t *) c->args.socket;
1142 for (i = 0; i < c->regions_num; i++)
1144 mr = &c->regions[i];
1149 if (mr->is_external)
1151 if (ms->get_external_region_addr == NULL)
1152 return MEMIF_ERR_INVAL_ARG;
1153 mr->addr = ms->get_external_region_addr (
1154 mr->region_size, mr->fd, c->private_ctx);
1159 return MEMIF_ERR_NO_SHMFD;
1162 mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE,
1163 MAP_SHARED, mr->fd, 0)) == MAP_FAILED)
1165 return memif_syscall_error_handler (errno);
1172 for (i = 0; i < c->rx_queues_num; i++)
1174 mq = &c->rx_queues[i];
1177 mq->ring = c->regions[mq->region].addr + mq->offset;
1178 if (mq->ring->cookie != MEMIF_COOKIE)
1180 DBG ("wrong cookie on rx ring %u", i);
1181 return MEMIF_ERR_COOKIE;
1183 mq->ring->head = mq->ring->tail = mq->last_head = mq->next_buf = 0;
1187 for (i = 0; i < c->tx_queues_num; i++)
1189 mq = &c->tx_queues[i];
1192 mq->ring = c->regions[mq->region].addr + mq->offset;
1193 if (mq->ring->cookie != MEMIF_COOKIE)
1195 DBG ("wrong cookie on tx ring %u", i);
1196 return MEMIF_ERR_COOKIE;
1198 mq->ring->head = mq->ring->tail = mq->last_head = mq->next_buf = 0;
1206 memif_add_region (memif_connection_t *conn, uint8_t has_buffers)
1209 memif_socket_t *ms = (memif_socket_t *) conn->args.socket;
1211 r = ms->args.realloc (conn->regions,
1212 sizeof (memif_region_t) * ++conn->regions_num);
1214 return MEMIF_ERR_NOMEM;
1217 r = &conn->regions[conn->regions_num - 1];
1218 memset (r, 0, sizeof (memif_region_t));
1220 if (has_buffers != 0)
1222 r->buffer_offset = 0;
1227 (conn->run_args.num_s2m_rings +
1228 conn->run_args.num_m2s_rings) * (sizeof (memif_ring_t) +
1229 sizeof (memif_desc_t) *
1231 run_args.log2_ring_size));
1234 r->region_size = (has_buffers == 0) ? r->buffer_offset : r->buffer_offset +
1235 conn->run_args.buffer_size * (1 << conn->run_args.log2_ring_size) *
1236 (conn->run_args.num_s2m_rings + conn->run_args.num_m2s_rings);
1238 if ((r->fd = memfd_create ("memif region 0", MFD_ALLOW_SEALING)) == -1)
1239 return memif_syscall_error_handler (errno);
1241 if ((fcntl (r->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
1242 return memif_syscall_error_handler (errno);
1244 if ((ftruncate (r->fd, r->region_size)) == -1)
1245 return memif_syscall_error_handler (errno);
1247 if ((r->addr = mmap (NULL, r->region_size, PROT_READ | PROT_WRITE,
1248 MAP_SHARED, r->fd, 0)) == MAP_FAILED)
1249 return memif_syscall_error_handler (errno);
1251 return MEMIF_ERR_SUCCESS;
1255 memif_init_queues (memif_connection_t *conn)
1259 memif_socket_t *ms = (memif_socket_t *) conn->args.socket;
1261 for (i = 0; i < conn->run_args.num_s2m_rings; i++)
1263 ring = memif_get_ring (conn, MEMIF_RING_S2M, i);
1264 DBG ("RING: %p I: %d", ring, i);
1265 ring->head = ring->tail = 0;
1266 ring->cookie = MEMIF_COOKIE;
1268 for (j = 0; j < (1 << conn->run_args.log2_ring_size); j++)
1270 uint16_t slot = i * (1 << conn->run_args.log2_ring_size) + j;
1271 ring->desc[j].region = 1;
1272 ring->desc[j].offset =
1273 conn->regions[1].buffer_offset +
1274 (uint32_t) (slot * conn->run_args.buffer_size);
1275 ring->desc[j].length = conn->run_args.buffer_size;
1278 for (i = 0; i < conn->run_args.num_m2s_rings; i++)
1280 ring = memif_get_ring (conn, MEMIF_RING_M2S, i);
1281 DBG ("RING: %p I: %d", ring, i);
1282 ring->head = ring->tail = 0;
1283 ring->cookie = MEMIF_COOKIE;
1285 for (j = 0; j < (1 << conn->run_args.log2_ring_size); j++)
1287 uint16_t slot = (i + conn->run_args.num_s2m_rings) *
1288 (1 << conn->run_args.log2_ring_size) + j;
1289 ring->desc[j].region = 1;
1290 ring->desc[j].offset =
1291 conn->regions[1].buffer_offset +
1292 (uint32_t) (slot * conn->run_args.buffer_size);
1293 ring->desc[j].length = conn->run_args.buffer_size;
1297 mq = (memif_queue_t *) ms->args.alloc (sizeof (memif_queue_t) *
1298 conn->run_args.num_s2m_rings);
1300 return MEMIF_ERR_NOMEM;
1304 for (x = 0; x < conn->run_args.num_s2m_rings; x++)
1306 if ((mq[x].int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
1307 return memif_syscall_error_handler (errno);
1309 mq[x].ring = memif_get_ring (conn, MEMIF_RING_S2M, x);
1310 DBG ("RING: %p I: %d", mq[x].ring, x);
1311 mq[x].log2_ring_size = conn->run_args.log2_ring_size;
1314 (void *) mq[x].ring - (void *) conn->regions[mq->region].addr;
1315 mq[x].last_head = mq[x].last_tail = 0;
1318 conn->tx_queues = mq;
1319 conn->tx_queues_num = conn->run_args.num_s2m_rings;
1321 mq = (memif_queue_t *) ms->args.alloc (sizeof (memif_queue_t) *
1322 conn->run_args.num_m2s_rings);
1324 return MEMIF_ERR_NOMEM;
1326 for (x = 0; x < conn->run_args.num_m2s_rings; x++)
1328 if ((mq[x].int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
1329 return memif_syscall_error_handler (errno);
1331 mq[x].ring = memif_get_ring (conn, MEMIF_RING_M2S, x);
1332 DBG ("RING: %p I: %d", mq[x].ring, x);
1333 mq[x].log2_ring_size = conn->run_args.log2_ring_size;
1336 (void *) mq[x].ring - (void *) conn->regions[mq->region].addr;
1337 mq[x].last_head = mq[x].last_tail = 0;
1340 conn->rx_queues = mq;
1341 conn->rx_queues_num = conn->run_args.num_m2s_rings;
1343 return MEMIF_ERR_SUCCESS;
1347 memif_init_regions_and_queues (memif_connection_t * conn)
1350 memif_socket_t *ms = (memif_socket_t *) conn->args.socket;
1352 /* region 0. rings */
1353 memif_add_region (conn, /* has_buffers */ 0);
1355 /* region 1. buffers */
1356 if (ms->add_external_region)
1358 r = (memif_region_t *) ms->args.realloc (
1359 conn->regions, sizeof (memif_region_t) * ++conn->regions_num);
1361 return MEMIF_ERR_NOMEM;
1364 conn->regions[1].region_size =
1365 conn->run_args.buffer_size * (1 << conn->run_args.log2_ring_size) *
1366 (conn->run_args.num_s2m_rings + conn->run_args.num_m2s_rings);
1367 conn->regions[1].buffer_offset = 0;
1368 ms->add_external_region (&conn->regions[1].addr,
1369 conn->regions[1].region_size,
1370 &conn->regions[1].fd, conn->private_ctx);
1371 conn->regions[1].is_external = 1;
1375 memif_add_region (conn, 1);
1378 memif_init_queues (conn);
1384 memif_set_next_free_buffer (memif_conn_handle_t conn, uint16_t qid,
1385 memif_buffer_t *buf)
1387 memif_connection_t *c = (memif_connection_t *) conn;
1388 if (EXPECT_FALSE (c == NULL))
1389 return MEMIF_ERR_NOCONN;
1390 if (EXPECT_FALSE (qid >= c->tx_queues_num))
1391 return MEMIF_ERR_QID;
1392 if (EXPECT_FALSE (buf == NULL))
1393 return MEMIF_ERR_INVAL_ARG;
1395 uint16_t ring_size, ns;
1396 memif_queue_t *mq = &c->tx_queues[qid];
1397 memif_ring_t *ring = mq->ring;
1399 ring_size = (1 << mq->log2_ring_size);
1400 if (c->args.is_master)
1401 ns = ring->head - mq->next_buf;
1403 ns = ring_size - mq->next_buf + ring->tail;
1405 if ((mq->next_buf - buf->desc_index) > ns)
1406 return MEMIF_ERR_INVAL_ARG;
1408 mq->next_buf = buf->desc_index;
1410 return MEMIF_ERR_SUCCESS;
1414 memif_buffer_enq_at_idx_internal (memif_queue_t *from_q, memif_queue_t *to_q,
1415 memif_buffer_t *buf, uint16_t slot)
1417 uint16_t from_mask = (1 << from_q->log2_ring_size) - 1;
1418 uint16_t to_mask = (1 << to_q->log2_ring_size) - 1;
1419 memif_desc_t *from_d, *to_d, tmp_d;
1421 /* Get the descriptors */
1422 from_d = &from_q->ring->desc[buf->desc_index & from_mask];
1423 to_d = &to_q->ring->desc[slot & to_mask];
1425 /* Swap descriptors */
1430 /* Update descriptor index and queue for clients buffer */
1431 buf->desc_index = slot;
1436 memif_buffer_requeue (memif_conn_handle_t conn, memif_buffer_t *buf_a,
1437 memif_buffer_t *buf_b)
1439 memif_connection_t *c = (memif_connection_t *) conn;
1440 if (EXPECT_FALSE (c == NULL))
1441 return MEMIF_ERR_NOCONN;
1442 if (EXPECT_FALSE (c->args.is_master))
1443 return MEMIF_ERR_INVAL_ARG;
1444 if ((buf_a == NULL) || (buf_b == NULL))
1445 return MEMIF_ERR_INVAL_ARG;
1448 /* store buf_a information */
1449 uint16_t index_a = buf_a->desc_index;
1450 memif_queue_t *mq_a = buf_a->queue;
1452 /* swap buffers, buf_a was updated with new desc_index and queue */
1453 memif_buffer_enq_at_idx_internal ((memif_queue_t *) buf_a->queue,
1454 (memif_queue_t *) buf_b->queue, buf_a,
1457 /* update buf_b desc_index and queue */
1458 buf_b->desc_index = index_a;
1459 buf_b->queue = mq_a;
1461 return MEMIF_ERR_SUCCESS;
1465 memif_buffer_enq_tx (memif_conn_handle_t conn, uint16_t qid,
1466 memif_buffer_t * bufs, uint16_t count,
1467 uint16_t * count_out)
1469 memif_connection_t *c = (memif_connection_t *) conn;
1470 if (EXPECT_FALSE (c == NULL))
1471 return MEMIF_ERR_NOCONN;
1472 if (EXPECT_FALSE (c->control_channel == NULL))
1473 return MEMIF_ERR_DISCONNECTED;
1474 if (EXPECT_FALSE (qid >= c->tx_queues_num))
1475 return MEMIF_ERR_QID;
1476 if (EXPECT_FALSE (!count_out))
1477 return MEMIF_ERR_INVAL_ARG;
1478 if (EXPECT_FALSE (c->args.is_master))
1479 return MEMIF_ERR_INVAL_ARG;
1481 memif_queue_t *mq = &c->tx_queues[qid];
1482 memif_ring_t *ring = mq->ring;
1484 uint16_t mask = (1 << mq->log2_ring_size) - 1;
1488 int err = MEMIF_ERR_SUCCESS; /* 0 */
1491 ring_size = (1 << mq->log2_ring_size);
1493 /* can only be called by slave */
1494 ns = ring_size - mq->next_buf + ring->tail;
1500 /* Swaps the descriptors, updates next_buf pointer and updates client
1503 memif_buffer_enq_at_idx_internal ((memif_queue_t *) b0->queue, mq, b0,
1506 mq->next_buf++; /* mark the buffer as allocated */
1513 DBG ("allocated: %u/%u bufs. Next buffer pointer %d", *count_out, count,
1518 DBG ("ring buffer full! qid: %u", qid);
1519 err = MEMIF_ERR_NOBUF_RING;
1526 memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
1527 memif_buffer_t * bufs, uint16_t count,
1528 uint16_t * count_out, uint16_t size)
1530 memif_connection_t *c = (memif_connection_t *) conn;
1531 if (EXPECT_FALSE (c == NULL))
1532 return MEMIF_ERR_NOCONN;
1533 if (EXPECT_FALSE (c->control_channel == NULL))
1534 return MEMIF_ERR_DISCONNECTED;
1536 (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1537 run_args.num_s2m_rings;
1538 if (EXPECT_FALSE (qid >= num))
1539 return MEMIF_ERR_QID;
1540 if (EXPECT_FALSE (!count_out))
1541 return MEMIF_ERR_INVAL_ARG;
1543 memif_socket_t *ms = (memif_socket_t *) c->args.socket;
1544 memif_queue_t *mq = &c->tx_queues[qid];
1545 memif_ring_t *ring = mq->ring;
1547 uint16_t mask = (1 << mq->log2_ring_size) - 1;
1550 int err = MEMIF_ERR_SUCCESS; /* 0 */
1551 uint16_t dst_left, src_left;
1552 uint16_t saved_count;
1553 uint16_t saved_next_buf;
1555 memif_buffer_t *saved_b;
1558 ring_size = (1 << mq->log2_ring_size);
1560 if (c->args.is_master)
1561 ns = ring->head - mq->next_buf;
1563 ns = ring_size - mq->next_buf + ring->tail;
1567 b0 = (bufs + *count_out);
1570 saved_count = count;
1571 saved_next_buf = mq->next_buf;
1573 b0->desc_index = mq->next_buf;
1574 ring->desc[mq->next_buf & mask].flags = 0;
1577 /* slave can produce buffer with original length */
1578 dst_left = (c->args.is_master) ? ring->desc[mq->next_buf & mask].length :
1579 c->run_args.buffer_size;
1584 if (EXPECT_FALSE (dst_left == 0))
1592 ring->desc[b0->desc_index & mask].flags |=
1593 MEMIF_DESC_FLAG_NEXT;
1594 b0->flags |= MEMIF_BUFFER_FLAG_NEXT;
1596 b0 = (bufs + *count_out);
1597 b0->desc_index = mq->next_buf;
1598 dst_left = (c->args.is_master) ?
1599 ring->desc[mq->next_buf & mask].length :
1600 c->run_args.buffer_size;
1601 ring->desc[mq->next_buf & mask].flags = 0;
1605 /* rollback allocated chain buffers */
1606 memset (saved_b, 0, sizeof (memif_buffer_t)
1607 * (saved_count - count + 1));
1608 *count_out -= saved_count - count;
1609 mq->next_buf = saved_next_buf;
1613 b0->len = memif_min (dst_left, src_left);
1615 /* slave resets buffer offset */
1616 if (c->args.is_master == 0)
1618 memif_desc_t *d = &ring->desc[slot & mask];
1619 if (ms->get_external_buffer_offset)
1620 d->offset = ms->get_external_buffer_offset (c->private_ctx);
1622 d->offset = d->offset - (d->offset % c->run_args.buffer_size);
1624 b0->data = memif_get_buffer (c, ring, mq->next_buf & mask);
1626 src_left -= b0->len;
1627 dst_left -= b0->len;
1638 DBG ("allocated: %u/%u bufs. Next buffer pointer %d", *count_out, count,
1643 DBG ("ring buffer full! qid: %u", qid);
1644 err = MEMIF_ERR_NOBUF_RING;
1651 memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count,
1654 memif_connection_t *c = (memif_connection_t *) conn;
1655 if (EXPECT_FALSE (c == NULL))
1656 return MEMIF_ERR_NOCONN;
1657 if (EXPECT_FALSE (c->control_channel == NULL))
1658 return MEMIF_ERR_DISCONNECTED;
1660 (c->args.is_master) ? c->run_args.num_s2m_rings : c->
1661 run_args.num_m2s_rings;
1662 if (EXPECT_FALSE (qid >= num))
1663 return MEMIF_ERR_QID;
1664 memif_socket_t *ms = (memif_socket_t *) c->args.socket;
1665 memif_queue_t *mq = &c->rx_queues[qid];
1666 memif_ring_t *ring = mq->ring;
1667 uint16_t mask = (1 << mq->log2_ring_size) - 1;
1668 uint16_t slot, counter = 0;
1670 if (c->args.is_master)
1672 MEMIF_MEMORY_BARRIER ();
1674 (ring->tail + count <=
1675 mq->last_head) ? ring->tail + count : mq->last_head;
1676 return MEMIF_ERR_SUCCESS;
1679 uint16_t head = ring->head;
1681 uint16_t ns = (1 << mq->log2_ring_size) - head + mq->last_tail;
1682 count = (count < ns) ? count : ns;
1685 while (counter < count)
1687 d = &ring->desc[slot & mask];
1689 d->length = c->run_args.buffer_size - headroom;
1690 if (ms->get_external_buffer_offset)
1691 d->offset = ms->get_external_buffer_offset (c->private_ctx);
1694 d->offset - (d->offset % c->run_args.buffer_size) + headroom;
1699 MEMIF_MEMORY_BARRIER ();
1702 return MEMIF_ERR_SUCCESS; /* 0 */
1706 memif_tx_burst (memif_conn_handle_t conn, uint16_t qid,
1707 memif_buffer_t * bufs, uint16_t count, uint16_t * tx)
1709 memif_connection_t *c = (memif_connection_t *) conn;
1710 if (EXPECT_FALSE (c == NULL))
1711 return MEMIF_ERR_NOCONN;
1712 if (EXPECT_FALSE (c->control_channel == NULL))
1713 return MEMIF_ERR_DISCONNECTED;
1715 (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1716 run_args.num_s2m_rings;
1717 if (EXPECT_FALSE (qid >= num))
1718 return MEMIF_ERR_QID;
1719 if (EXPECT_FALSE (!tx))
1720 return MEMIF_ERR_INVAL_ARG;
1722 memif_queue_t *mq = &c->tx_queues[qid];
1723 memif_ring_t *ring = mq->ring;
1724 uint16_t mask = (1 << mq->log2_ring_size) - 1;
1727 int64_t data_offset;
1729 int err = MEMIF_ERR_SUCCESS;
1731 if (EXPECT_FALSE (count == 0))
1732 return MEMIF_ERR_SUCCESS;
1735 if (c->args.is_master)
1743 /* set error to MEMIF_ERR_INVAL_ARG and finish the sending process
1745 if ((b0->desc_index & mask) != (index & mask))
1747 err = MEMIF_ERR_INVAL_ARG;
1750 d = &ring->desc[b0->desc_index & mask];
1751 d->length = b0->len;
1753 ((b0->flags & MEMIF_BUFFER_FLAG_NEXT) == 1) ? MEMIF_DESC_FLAG_NEXT : 0;
1754 if (!c->args.is_master)
1757 d->offset = d->offset - (d->offset % c->run_args.buffer_size);
1758 // calculate offset from user data
1759 data_offset = b0->data - (d->offset + c->regions[d->region].addr);
1760 if (data_offset != 0)
1762 /* verify data offset and buffer length */
1763 if ((data_offset < 0) ||
1764 ((data_offset + b0->len) > c->run_args.buffer_size))
1766 DBG ("slot: %d, data_offset: %ld, length: %d",
1767 b0->desc_index & mask, data_offset, b0->len);
1768 err = MEMIF_ERR_INVAL_ARG;
1771 d->offset += data_offset;
1775 #ifdef MEMIF_DBG_SHM
1776 printf ("offset: %-6d\n", ring->desc[b0->desc_index & mask].offset);
1777 printf ("data: %p\n",
1778 memif_get_buffer (c, ring, b0->desc_index & mask));
1779 printf ("index: %u\n", b0->desc_index);
1780 print_bytes (memif_get_buffer (c, ring, b0->desc_index & mask),
1781 ring->desc[b0->desc_index & mask].length, DBG_TX_BUF);
1782 #endif /* MEMIF_DBG_SHM */
1790 MEMIF_MEMORY_BARRIER ();
1791 if (c->args.is_master)
1792 ring->tail = b0->desc_index + 1;
1794 ring->head = b0->desc_index + 1;
1796 if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0)
1799 int r = write (mq->int_fd, &a, sizeof (a));
1801 return MEMIF_ERR_INT_WRITE;
1808 memif_rx_burst (memif_conn_handle_t conn, uint16_t qid,
1809 memif_buffer_t * bufs, uint16_t count, uint16_t * rx)
1811 memif_connection_t *c = (memif_connection_t *) conn;
1812 if (EXPECT_FALSE (c == NULL))
1813 return MEMIF_ERR_NOCONN;
1814 if (EXPECT_FALSE (c->control_channel == NULL))
1815 return MEMIF_ERR_DISCONNECTED;
1817 (c->args.is_master) ? c->run_args.num_s2m_rings : c->
1818 run_args.num_m2s_rings;
1819 if (EXPECT_FALSE (qid >= num))
1820 return MEMIF_ERR_QID;
1821 if (EXPECT_FALSE (!rx))
1822 return MEMIF_ERR_INVAL_ARG;
1824 memif_queue_t *mq = &c->rx_queues[qid];
1825 memif_ring_t *ring = mq->ring;
1826 uint16_t cur_slot, last_slot;
1828 uint16_t mask = (1 << mq->log2_ring_size) - 1;
1835 cur_slot = (c->args.is_master) ? mq->last_head : mq->last_tail;
1836 last_slot = (c->args.is_master) ? ring->head : ring->tail;
1837 if (cur_slot == last_slot)
1839 r = read (mq->int_fd, &b, sizeof (b));
1840 if (EXPECT_FALSE ((r == -1) && (errno != EAGAIN)))
1841 return memif_syscall_error_handler (errno);
1843 return MEMIF_ERR_SUCCESS;
1846 ns = last_slot - cur_slot;
1852 b0->desc_index = cur_slot;
1853 b0->data = memif_get_buffer (c, ring, cur_slot & mask);
1854 b0->len = ring->desc[cur_slot & mask].length;
1856 /* slave resets buffer length */
1857 if (c->args.is_master == 0)
1859 ring->desc[cur_slot & mask].length = c->run_args.buffer_size;
1862 if (ring->desc[cur_slot & mask].flags & MEMIF_DESC_FLAG_NEXT)
1864 b0->flags |= MEMIF_BUFFER_FLAG_NEXT;
1865 ring->desc[cur_slot & mask].flags &= ~MEMIF_DESC_FLAG_NEXT;
1869 #ifdef MEMIF_DBG_SHM
1870 printf ("data: %p\n", b0->data);
1871 printf ("index: %u\n", b0->desc_index);
1872 printf ("queue: %p\n", b0->queue);
1873 print_bytes (b0->data, b0->len, DBG_RX_BUF);
1874 #endif /* MEMIF_DBG_SHM */
1882 if (c->args.is_master)
1883 mq->last_head = cur_slot;
1885 mq->last_tail = cur_slot;
1889 DBG ("not enough buffers!");
1890 return MEMIF_ERR_NOBUF;
1893 r = read (mq->int_fd, &b, sizeof (b));
1894 if (EXPECT_FALSE ((r == -1) && (errno != EAGAIN)))
1895 return memif_syscall_error_handler (errno);
1897 return MEMIF_ERR_SUCCESS; /* 0 */
1901 memif_get_details (memif_conn_handle_t conn, memif_details_t * md,
1902 char *buf, ssize_t buflen)
1904 memif_connection_t *c = (memif_connection_t *) conn;
1906 int err = MEMIF_ERR_SUCCESS, i;
1910 return MEMIF_ERR_NOCONN;
1912 ms = (memif_socket_t *) c->args.socket;
1914 l1 = strlen ((char *) c->args.interface_name);
1915 if (l0 + l1 < buflen)
1918 (uint8_t *) strcpy (buf + l0, (char *) c->args.interface_name);
1922 err = MEMIF_ERR_NOBUF_DET;
1924 l1 = strlen ((char *) ms->args.app_name);
1925 if (l0 + l1 < buflen)
1928 (uint8_t *) strcpy (buf + l0, (char *) ms->args.app_name);
1932 err = MEMIF_ERR_NOBUF_DET;
1934 l1 = strlen ((char *) c->remote_if_name);
1935 if (l0 + l1 < buflen)
1937 md->remote_if_name =
1938 (uint8_t *) strcpy (buf + l0, (char *) c->remote_if_name);
1942 err = MEMIF_ERR_NOBUF_DET;
1944 l1 = strlen ((char *) c->remote_name);
1945 if (l0 + l1 < buflen)
1947 md->remote_inst_name =
1948 (uint8_t *) strcpy (buf + l0, (char *) c->remote_name);
1952 err = MEMIF_ERR_NOBUF_DET;
1954 md->id = c->args.interface_id;
1956 if (strlen ((char *) c->args.secret) > 0)
1958 l1 = strlen ((char *) c->args.secret);
1959 if (l0 + l1 < buflen)
1961 md->secret = (uint8_t *) strcpy (buf + l0, (char *) c->args.secret);
1965 err = MEMIF_ERR_NOBUF_DET;
1968 md->role = (c->args.is_master) ? 0 : 1;
1969 md->mode = c->args.mode;
1972 if (l0 + l1 < buflen)
1974 md->socket_path = (uint8_t *) memcpy (buf + l0, ms->args.path, 108);
1978 err = MEMIF_ERR_NOBUF_DET;
1980 l1 = strlen ((char *) c->remote_disconnect_string);
1981 if (l0 + l1 < buflen)
1984 (uint8_t *) strcpy (buf + l0, (char *) c->remote_disconnect_string);
1988 err = MEMIF_ERR_NOBUF_DET;
1990 md->regions_num = c->regions_num;
1991 l1 = sizeof (memif_region_details_t) * md->regions_num;
1992 if (l0 + l1 <= buflen)
1994 md->regions = (memif_region_details_t *) (buf + l0);
1995 for (i = 0; i < md->regions_num; i++)
1997 md->regions[i].index = i;
1998 md->regions[i].addr = c->regions[i].addr;
1999 md->regions[i].size = c->regions[i].region_size;
2000 md->regions[i].fd = c->regions[i].fd;
2001 md->regions[i].is_external = c->regions[i].is_external;
2006 err = MEMIF_ERR_NOBUF_DET;
2009 (c->args.is_master) ? c->run_args.num_s2m_rings : c->
2010 run_args.num_m2s_rings;
2012 l1 = sizeof (memif_queue_details_t) * md->rx_queues_num;
2013 if (l0 + l1 <= buflen)
2015 md->rx_queues = (memif_queue_details_t *) (buf + l0);
2016 for (i = 0; i < md->rx_queues_num; i++)
2018 md->rx_queues[i].region = c->rx_queues[i].region;
2019 md->rx_queues[i].qid = i;
2020 md->rx_queues[i].ring_size = (1 << c->rx_queues[i].log2_ring_size);
2021 md->rx_queues[i].flags = c->rx_queues[i].ring->flags;
2022 md->rx_queues[i].head = c->rx_queues[i].ring->head;
2023 md->rx_queues[i].tail = c->rx_queues[i].ring->tail;
2024 md->rx_queues[i].buffer_size = c->run_args.buffer_size;
2029 err = MEMIF_ERR_NOBUF_DET;
2032 (c->args.is_master) ? c->run_args.num_m2s_rings : c->
2033 run_args.num_s2m_rings;
2035 l1 = sizeof (memif_queue_details_t) * md->tx_queues_num;
2036 if (l0 + l1 <= buflen)
2038 md->tx_queues = (memif_queue_details_t *) (buf + l0);
2039 for (i = 0; i < md->tx_queues_num; i++)
2041 md->tx_queues[i].region = c->tx_queues[i].region;
2042 md->tx_queues[i].qid = i;
2043 md->tx_queues[i].ring_size = (1 << c->tx_queues[i].log2_ring_size);
2044 md->tx_queues[i].flags = c->tx_queues[i].ring->flags;
2045 md->tx_queues[i].head = c->tx_queues[i].ring->head;
2046 md->tx_queues[i].tail = c->tx_queues[i].ring->tail;
2047 md->tx_queues[i].buffer_size = c->run_args.buffer_size;
2052 err = MEMIF_ERR_NOBUF_DET;
2054 /* This is not completely true, clients should relay on
2055 * on_connect/on_disconnect callbacks */
2056 md->link_up_down = (c->control_channel != NULL) ? 1 : 0;
2062 memif_get_queue_efd (memif_conn_handle_t conn, uint16_t qid, int *efd)
2064 memif_connection_t *c = (memif_connection_t *) conn;
2069 return MEMIF_ERR_NOCONN;
2070 if (c->control_channel == NULL)
2071 return MEMIF_ERR_DISCONNECTED;
2074 (c->args.is_master) ? c->run_args.num_s2m_rings : c->
2075 run_args.num_m2s_rings;
2077 return MEMIF_ERR_QID;
2079 *efd = c->rx_queues[qid].int_fd;
2081 return MEMIF_ERR_SUCCESS;