1 // Copyright (c) 2017 Cisco and/or its affiliates.
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at:
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
15 // +build !windows,!darwin
26 logger "github.com/sirupsen/logrus"
36 #include <sys/eventfd.h>
37 #include <libmemif.h> // <-- VPP must be installed!
40 #ifndef MEMIF_HAVE_CANCEL_POLL_EVENT
41 // memif_cancel_poll_event that simply returns ErrUnsupported.
43 memif_cancel_poll_event ()
45 return 102; // ErrUnsupported
49 // govpp_memif_conn_args_t replaces fixed sized arrays with C-strings which
50 // are much easier to work with in cgo.
53 memif_socket_handle_t socket;
55 uint8_t num_s2m_rings;
56 uint8_t num_m2s_rings;
58 uint8_t log2_ring_size;
60 uint32_t interface_id;
62 memif_interface_mode_t mode;
63 } govpp_memif_conn_args_t;
65 // govpp_memif_details_t replaces strings represented with (uint8_t *)
66 // to the standard and easy to work with in cgo: (char *)
72 char *remote_inst_name;
77 char *socket_filename;
79 memif_region_details_t *regions;
80 uint8_t rx_queues_num;
81 uint8_t tx_queues_num;
82 memif_queue_details_t *rx_queues;
83 memif_queue_details_t *tx_queues;
85 } govpp_memif_details_t;
87 extern int go_on_connect_callback(void *privateCtx);
88 extern int go_on_disconnect_callback(void *privateCtx);
90 // Callbacks strip the connection handle away.
93 govpp_on_connect_callback(memif_conn_handle_t conn, void *private_ctx)
95 return go_on_connect_callback(private_ctx);
99 govpp_on_disconnect_callback(memif_conn_handle_t conn, void *private_ctx)
101 return go_on_disconnect_callback(private_ctx);
104 // govpp_memif_create uses govpp_memif_conn_args_t.
106 govpp_memif_create (memif_conn_handle_t *conn, govpp_memif_conn_args_t *go_args,
109 memif_conn_args_t args;
110 memset (&args, 0, sizeof (args));
111 args.socket = (char *)go_args->socket;
112 if (go_args->secret != NULL)
114 strncpy ((char *)args.secret, go_args->secret,
115 sizeof (args.secret) - 1);
117 args.num_s2m_rings = go_args->num_s2m_rings;
118 args.num_m2s_rings = go_args->num_m2s_rings;
119 args.buffer_size = go_args->buffer_size;
120 args.log2_ring_size = go_args->log2_ring_size;
121 args.is_master = go_args->is_master;
122 args.interface_id = go_args->interface_id;
123 if (go_args->interface_name != NULL)
125 strncpy ((char *)args.interface_name, go_args->interface_name,
126 sizeof(args.interface_name) - 1);
128 args.mode = go_args->mode;
130 return memif_create(conn, &args, govpp_on_connect_callback,
131 govpp_on_disconnect_callback, NULL,
136 govpp_memif_create_socket (memif_socket_handle_t *sock, char *filename)
138 return memif_create_socket(sock, filename, NULL);
141 // govpp_memif_get_details keeps reallocating buffer until it is large enough.
142 // The buffer is returned to be deallocated when it is no longer needed.
144 govpp_memif_get_details (memif_conn_handle_t conn, govpp_memif_details_t *govpp_md,
148 size_t buflen = 1 << 7;
149 char *buffer = NULL, *new_buffer = NULL;
150 memif_details_t md = {0};
153 // initial malloc (256 bytes) or realloc
155 new_buffer = realloc(buffer, buflen);
156 if (new_buffer == NULL)
159 return MEMIF_ERR_NOMEM;
162 // try to get details
163 rv = memif_get_details(conn, &md, buffer, buflen);
164 } while (rv == MEMIF_ERR_NOBUF_DET);
169 govpp_md->if_name = (char *)md.if_name;
170 govpp_md->inst_name = (char *)md.inst_name;
171 govpp_md->remote_if_name = (char *)md.remote_if_name;
172 govpp_md->remote_inst_name = (char *)md.remote_inst_name;
173 govpp_md->id = md.id;
174 govpp_md->secret = (char *)md.secret;
175 govpp_md->role = md.role;
176 govpp_md->mode = md.mode;
177 govpp_md->socket_filename = (char *)md.socket_filename;
178 govpp_md->regions_num = md.regions_num;
179 govpp_md->regions = md.regions;
180 govpp_md->rx_queues_num = md.rx_queues_num;
181 govpp_md->tx_queues_num = md.tx_queues_num;
182 govpp_md->rx_queues = md.rx_queues;
183 govpp_md->tx_queues = md.tx_queues;
184 govpp_md->link_up_down = md.link_up_down;
191 // Used to avoid cumbersome tricks that use unsafe.Pointer() + unsafe.Sizeof()
192 // or even cast C-array directly into Go-slice.
193 static memif_queue_details_t
194 govpp_get_rx_queue_details (govpp_memif_details_t *md, int index)
196 return md->rx_queues[index];
199 // Used to avoid cumbersome tricks that use unsafe.Pointer() + unsafe.Sizeof()
200 // or even cast C-array directly into Go-slice.
201 static memif_queue_details_t
202 govpp_get_tx_queue_details (govpp_memif_details_t *md, int index)
204 return md->tx_queues[index];
207 // Copy packet data into the selected buffer with splitting when necessary
209 govpp_copy_packet_data(memif_buffer_t *buffers, uint16_t allocated, int bufIndex, void *packetData, uint16_t packetSize)
214 buffers[bufIndex].len = (packetSize > buffers[bufIndex].len ? buffers[bufIndex].len : packetSize);
215 void * curData = (packetData + dataOffset);
216 memcpy(buffers[bufIndex].data, curData, (size_t)buffers[bufIndex].len);
217 dataOffset += buffers[bufIndex].len;
219 packetSize -= buffers[bufIndex].len;
220 } while(packetSize > 0 && bufIndex < allocated && buffers[bufIndex].flags & MEMIF_BUFFER_FLAG_NEXT > 0);
223 // Get packet data from the selected buffer.
224 // Used to avoid an ugly unsafe.Pointer() + unsafe.Sizeof().
226 govpp_get_packet_data(memif_buffer_t *buffers, int index, int *size)
228 *size = (int)buffers[index].len;
229 return buffers[index].data;
232 // Checks if memif buffer is chained
234 govpp_is_buffer_chained(memif_buffer_t *buffers, int index)
236 return buffers[index].flags & MEMIF_BUFFER_FLAG_NEXT;
239 // Allocate memif buffers and return pointer to next free buffer
241 govpp_memif_buffer_alloc(memif_conn_handle_t conn, uint16_t qid,
242 memif_buffer_t * bufs, uint16_t offset, memif_buffer_t ** nextFreeBuf,
243 uint16_t count, uint16_t * count_out, uint16_t size)
245 memif_buffer_t * offsetBufs = (bufs + offset);
246 int err = memif_buffer_alloc(conn, qid, offsetBufs, count, count_out, size);
247 *count_out += offset;
248 *nextFreeBuf = offsetBufs;
255 // IfMode represents the mode (layer/behaviour) in which the interface operates.
259 // IfModeEthernet tells memif to operate on the L2 layer.
260 IfModeEthernet IfMode = iota
262 // IfModeIP tells memif to operate on the L3 layer.
265 // IfModePuntInject tells memif to behave as Inject/Punt interface.
269 // RxMode is used to switch between polling and interrupt for RX.
273 // RxModeInterrupt tells libmemif to send interrupt signal when data are available.
274 RxModeInterrupt RxMode = iota
276 // RxModePolling means that the user needs to explicitly poll for data on RX
281 // RawPacketData represents raw packet data. libmemif doesn't care what the
282 // actual content is, it only manipulates with raw bytes.
283 type RawPacketData []byte
285 // MemifMeta is used to store a basic memif metadata needed for identification
286 // and connection establishment.
287 type MemifMeta struct {
288 // IfName is the interface name. Has to be unique across all created memifs.
289 // Interface name is truncated if needed to have no more than 32 characters.
292 // InstanceName identifies the endpoint. If omitted, the application
293 // name passed to Init() will be used instead.
294 // Instance name is truncated if needed to have no more than 32 characters.
297 // ConnID is a connection ID used to match opposite sides of the memif
301 // SocketFilename is the filename of the AF_UNIX socket through which
302 // the connection is established.
303 // The string is truncated if neede to fit into sockaddr_un.sun_path
304 // (108 characters on Linux).
305 SocketFilename string
307 // Secret must be the same on both sides for the authentication to succeed.
308 // Empty string is allowed.
309 // The secret is truncated if needed to have no more than 24 characters.
312 // IsMaster is set to true if memif operates in the Master mode.
315 // Mode is the mode (layer/behaviour) in which the memif operates.
319 // MemifShmSpecs is used to store the specification of the shared memory segment
320 // used by memif to send/receive packets.
321 type MemifShmSpecs struct {
322 // NumRxQueues is the number of Rx queues.
323 // Default is 1 (used if the value is 0).
326 // NumTxQueues is the number of Tx queues.
327 // Default is 1 (used if the value is 0).
330 // BufferSize is the size of the buffer to hold one packet, or a single
331 // fragment of a jumbo frame. Default is 2048 (used if the value is 0).
334 // Log2RingSize is the number of items in the ring represented through
335 // the logarithm base 2.
336 // Default is 10 (used if the value is 0).
340 // MemifConfig is the memif configuration.
341 // Used as the input argument to CreateInterface().
342 // It is the slave's config that mostly decides the parameters of the connection,
343 // but master may limit some of the quantities if needed (based on the memif
344 // protocol or master's configuration)
345 type MemifConfig struct {
350 // ConnUpdateCallback is a callback type declaration used with callbacks
351 // related to connection status changes.
352 type ConnUpdateCallback func(memif *Memif) (err error)
354 // MemifCallbacks is a container for all callbacks provided by memif.
355 // Any callback can be nil, in which case it will be simply skipped.
356 // Important: Do not call CreateInterface() or Memif.Close() from within a callback
357 // or a deadlock will occur. Instead send signal through a channel to another
358 // go routine which will be able to create/remove memif interface(s).
359 type MemifCallbacks struct {
360 // OnConnect is triggered when a connection for a given memif was established.
361 OnConnect ConnUpdateCallback
363 // OnDisconnect is triggered when a connection for a given memif was lost.
364 OnDisconnect ConnUpdateCallback
367 // Memif represents a single memif interface. It provides methods to send/receive
368 // packets in bursts in either the polling mode or in the interrupt mode with
369 // the help of golang channels.
373 // Per-library references
374 ifIndex int // index used in the Go-libmemif context (Context.memifs)
375 cHandle C.memif_conn_handle_t // connection handle used in C-libmemif
376 sHandle C.memif_socket_handle_t // socket handle used in C-libmemif
379 callbacks *MemifCallbacks
382 intCh chan uint8 // memif-global interrupt channel (value = queue ID)
383 queueIntCh []chan struct{} // per RX queue interrupt channel
386 ringSize int // number of items in each ring
387 bufferSize int // max buffer size
388 stopQPollFd int // event file descriptor used to stop pollRxQueue-s
389 wg sync.WaitGroup // wait group for all pollRxQueue-s
390 rxQueueBufs []CPacketBuffers // an array of C-libmemif packet buffers for each RX queue
391 txQueueBufs []CPacketBuffers // an array of C-libmemif packet buffers for each TX queue
394 // MemifDetails provides a detailed runtime information about a memif interface.
395 type MemifDetails struct {
400 // MemifConnDetails provides a detailed runtime information about a memif
402 type MemifConnDetails struct {
403 // RemoteIfName is the name of the memif on the opposite side.
405 // RemoteInstanceName is the name of the endpoint on the opposite side.
406 RemoteInstanceName string
407 // HasLink is true if the connection has link (= is established and functional).
409 // RxQueues contains details for each Rx queue.
410 RxQueues []MemifQueueDetails
411 // TxQueues contains details for each Tx queue.
412 TxQueues []MemifQueueDetails
415 // MemifQueueDetails provides a detailed runtime information about a memif queue.
416 // Queue = Ring + the associated buffers (one directional).
417 type MemifQueueDetails struct {
418 // QueueID is the ID of the queue.
420 // RingSize is the number of slots in the ring (not logarithmic).
422 // BufferSize is the size of each buffer pointed to from the ring slots.
424 /* Further ring information TO-BE-ADDED when C-libmemif supports them. */
427 // CPacketBuffers stores an array of memif buffers for use with TxBurst or RxBurst.
428 type CPacketBuffers struct {
429 buffers *C.memif_buffer_t
431 rxChainBuf []RawPacketData
434 // Context is a global Go-libmemif runtime context.
435 type Context struct {
438 memifs map[int] /* ifIndex */ *Memif /* slice of all active memif interfaces */
441 wg sync.WaitGroup /* wait-group for pollEvents() */
444 type txPacketBuffer struct {
445 packets []RawPacketData
450 // logger used by the adapter.
453 // Global Go-libmemif context.
454 context = &Context{initialized: false}
457 // init initializes global logger, which logs debug level messages to stdout.
461 log.Level = logger.DebugLevel
464 // SetLogger changes the logger for Go-libmemif to the provided one.
465 // The logger is not used for logging of C-libmemif.
466 func SetLogger(l *logger.Logger) {
470 // Init initializes the libmemif library. Must by called exactly once and before
471 // any libmemif functions. Do not forget to call Cleanup() before exiting
473 // <appName> should be a human-readable string identifying your application.
474 // For example, VPP returns the version information ("show version" from VPP CLI).
475 func Init(appName string) error {
477 defer context.lock.Unlock()
479 if context.initialized {
480 return ErrAlreadyInit
483 log.Debug("Initializing libmemif library")
485 // Initialize C-libmemif.
488 errCode = int(C.memif_init(nil, nil, nil, nil, nil))
490 appName := C.CString(appName)
491 defer C.free(unsafe.Pointer(appName))
492 errCode = int(C.memif_init(nil, appName, nil, nil, nil))
494 err := getMemifError(errCode)
499 // Initialize the map of memory interfaces.
500 context.memifs = make(map[int]*Memif)
502 // Start event polling.
506 context.initialized = true
507 log.Debug("libmemif library was initialized")
511 // Cleanup cleans up all the resources allocated by libmemif.
512 func Cleanup() error {
514 defer context.lock.Unlock()
516 if !context.initialized {
520 log.Debug("Closing libmemif library")
522 // Delete all active interfaces.
523 for _, memif := range context.memifs {
527 // Stop the event loop (if supported by C-libmemif).
528 errCode := C.memif_cancel_poll_event()
529 err := getMemifError(int(errCode))
531 log.Debug("Waiting for pollEvents() to stop...")
533 log.Debug("pollEvents() has stopped...")
535 log.WithField("err", err).Debug("NOT Waiting for pollEvents to stop...")
538 // Run cleanup for C-libmemif.
539 err = getMemifError(int(C.memif_cleanup()))
541 context.initialized = false
542 log.Debug("libmemif library was closed")
547 // CreateInterface creates a new memif interface with the given configuration.
548 // The same callbacks can be used with multiple memifs. The first callback input
549 // argument (*Memif) can be used to tell which memif the callback was triggered for.
550 // The method is thread-safe.
551 func CreateInterface(config *MemifConfig, callbacks *MemifCallbacks) (memif *Memif, err error) {
553 defer context.lock.Unlock()
555 if !context.initialized {
556 return nil, ErrNotInit
559 log.WithField("ifName", config.IfName).Debug("Creating a new memif interface")
561 log2RingSize := config.Log2RingSize
562 if log2RingSize == 0 {
566 bufferSize := config.BufferSize
571 // Create memif-wrapper for Go-libmemif.
573 MemifMeta: config.MemifMeta,
574 callbacks: &MemifCallbacks{},
575 ifIndex: context.nextMemifIndex,
576 ringSize: 1 << log2RingSize,
577 bufferSize: int(bufferSize),
580 // Initialize memif callbacks.
581 if callbacks != nil {
582 memif.callbacks.OnConnect = callbacks.OnConnect
583 memif.callbacks.OnDisconnect = callbacks.OnDisconnect
586 // Initialize memif-global interrupt channel.
587 memif.intCh = make(chan uint8, 1<<6)
589 // Initialize event file descriptor for stopping Rx/Tx queue polling.
590 memif.stopQPollFd = int(C.eventfd(0, C.EFD_NONBLOCK))
591 if memif.stopQPollFd < 0 {
592 return nil, ErrSyscall
595 // Initialize memif input arguments.
596 args := &C.govpp_memif_conn_args_t{}
597 // - socket file name
598 if config.SocketFilename != "" {
599 log.WithField("name", config.SocketFilename).Debug("A new memif socket was created")
600 errCode := C.govpp_memif_create_socket(&memif.sHandle, C.CString(config.SocketFilename))
601 if getMemifError(int(errCode)) != nil {
605 args.socket = memif.sHandle
607 args.interface_id = C.uint32_t(config.ConnID)
609 if config.IfName != "" {
610 args.interface_name = C.CString(config.IfName)
611 defer C.free(unsafe.Pointer(args.interface_name))
616 args.mode = C.MEMIF_INTERFACE_MODE_ETHERNET
618 args.mode = C.MEMIF_INTERFACE_MODE_IP
619 case IfModePuntInject:
620 args.mode = C.MEMIF_INTERFACE_MODE_PUNT_INJECT
622 args.mode = C.MEMIF_INTERFACE_MODE_ETHERNET
625 if config.Secret != "" {
626 args.secret = C.CString(config.Secret)
627 defer C.free(unsafe.Pointer(args.secret))
629 // - master/slave flag + number of Rx/Tx queues
631 args.num_s2m_rings = C.uint8_t(config.NumRxQueues)
632 args.num_m2s_rings = C.uint8_t(config.NumTxQueues)
633 args.is_master = C.uint8_t(1)
635 args.num_s2m_rings = C.uint8_t(config.NumTxQueues)
636 args.num_m2s_rings = C.uint8_t(config.NumRxQueues)
637 args.is_master = C.uint8_t(0)
640 args.buffer_size = C.uint16_t(config.BufferSize)
641 // - log_2(ring size)
642 args.log2_ring_size = C.uint8_t(config.Log2RingSize)
644 // Create memif in C-libmemif.
645 errCode := C.govpp_memif_create(&memif.cHandle, args, unsafe.Pointer(uintptr(memif.ifIndex)))
646 if getMemifError(int(errCode)) != nil {
650 // Register the new memif.
651 context.memifs[memif.ifIndex] = memif
652 context.nextMemifIndex++
653 log.WithField("ifName", config.IfName).Debug("A new memif interface was created")
658 // GetInterruptChan returns a channel which is continuously being filled with
659 // IDs of queues with data ready to be received.
660 // Since there is only one interrupt signal sent for an entire burst of packets,
661 // an interrupt handling routine should repeatedly call RxBurst() until
662 // the function returns an empty slice of packets. This way it is ensured
663 // that there are no packets left on the queue unread when the interrupt signal
665 // The method is thread-safe.
666 func (memif *Memif) GetInterruptChan() (ch <-chan uint8 /* queue ID */) {
670 // GetQueueInterruptChan returns an empty-data channel which fires every time
671 // there are data to read on a given queue.
672 // It is only valid to call this function if memif is in the connected state.
673 // Channel is automatically closed when the connection goes down (but after
674 // the user provided callback OnDisconnect has executed).
675 // Since there is only one interrupt signal sent for an entire burst of packets,
676 // an interrupt handling routine should repeatedly call RxBurst() until
677 // the function returns an empty slice of packets. This way it is ensured
678 // that there are no packets left on the queue unread when the interrupt signal
680 // The method is thread-safe.
681 func (memif *Memif) GetQueueInterruptChan(queueID uint8) (ch <-chan struct{}, err error) {
682 if int(queueID) >= len(memif.queueIntCh) {
683 return nil, ErrQueueID
685 return memif.queueIntCh[queueID], nil
688 // SetRxMode allows to switch between the interrupt and the polling mode for Rx.
689 // The method is thread-safe.
690 func (memif *Memif) SetRxMode(queueID uint8, rxMode RxMode) (err error) {
691 var cRxMode C.memif_rx_mode_t
693 case RxModeInterrupt:
694 cRxMode = C.MEMIF_RX_MODE_INTERRUPT
696 cRxMode = C.MEMIF_RX_MODE_POLLING
698 cRxMode = C.MEMIF_RX_MODE_INTERRUPT
700 errCode := C.memif_set_rx_mode(memif.cHandle, cRxMode, C.uint16_t(queueID))
701 return getMemifError(int(errCode))
704 // GetDetails returns a detailed runtime information about this memif.
705 // The method is thread-safe.
706 func (memif *Memif) GetDetails() (details *MemifDetails, err error) {
707 cDetails := C.govpp_memif_details_t{}
710 // Get memif details from C-libmemif.
711 errCode := C.govpp_memif_get_details(memif.cHandle, &cDetails, &buf)
712 err = getMemifError(int(errCode))
716 defer C.free(unsafe.Pointer(buf))
718 // Convert details from C to Go.
719 details = &MemifDetails{}
721 details.IfName = C.GoString(cDetails.if_name)
722 details.InstanceName = C.GoString(cDetails.inst_name)
723 details.ConnID = uint32(cDetails.id)
724 details.SocketFilename = C.GoString(cDetails.socket_filename)
725 if cDetails.secret != nil {
726 details.Secret = C.GoString(cDetails.secret)
728 details.IsMaster = cDetails.role == C.uint8_t(0)
729 switch cDetails.mode {
730 case C.MEMIF_INTERFACE_MODE_ETHERNET:
731 details.Mode = IfModeEthernet
732 case C.MEMIF_INTERFACE_MODE_IP:
733 details.Mode = IfModeIP
734 case C.MEMIF_INTERFACE_MODE_PUNT_INJECT:
735 details.Mode = IfModePuntInject
737 details.Mode = IfModeEthernet
739 // - connection details:
740 details.RemoteIfName = C.GoString(cDetails.remote_if_name)
741 details.RemoteInstanceName = C.GoString(cDetails.remote_inst_name)
742 details.HasLink = cDetails.link_up_down == C.uint8_t(1)
745 for i = 0; i < uint8(cDetails.rx_queues_num); i++ {
746 cRxQueue := C.govpp_get_rx_queue_details(&cDetails, C.int(i))
747 queueDetails := MemifQueueDetails{
748 QueueID: uint8(cRxQueue.qid),
749 RingSize: uint32(cRxQueue.ring_size),
750 BufferSize: uint16(cRxQueue.buffer_size),
752 details.RxQueues = append(details.RxQueues, queueDetails)
755 for i = 0; i < uint8(cDetails.tx_queues_num); i++ {
756 cTxQueue := C.govpp_get_tx_queue_details(&cDetails, C.int(i))
757 queueDetails := MemifQueueDetails{
758 QueueID: uint8(cTxQueue.qid),
759 RingSize: uint32(cTxQueue.ring_size),
760 BufferSize: uint16(cTxQueue.buffer_size),
762 details.TxQueues = append(details.TxQueues, queueDetails)
768 // TxBurst is used to send multiple packets in one call into a selected queue.
769 // The actual number of packets sent may be smaller and is returned as <count>.
770 // The method is non-blocking even if the ring is full and no packet can be sent.
771 // It is only valid to call this function if memif is in the connected state.
772 // Multiple TxBurst-s can run concurrently provided that each targets a different
774 func (memif *Memif) TxBurst(queueID uint8, packets []RawPacketData) (count uint16, err error) {
775 if len(packets) == 0 {
779 if int(queueID) >= len(memif.txQueueBufs) {
784 buffers := make([]*txPacketBuffer, 0)
785 cQueueID := C.uint16_t(queueID)
787 for _, packet := range packets {
788 packetLen := len(packet)
789 log.Debugf("%v - preparing packet with len %v", cQueueID, packetLen)
791 if packetLen > memif.bufferSize {
792 // Create jumbo buffer
793 buffer := &txPacketBuffer{
795 packets: []RawPacketData{packet},
798 buffers = append(buffers, buffer)
800 // Increment bufCount by number of splits in this jumbo
801 bufCount += (buffer.size + memif.bufferSize - 1) / memif.bufferSize
803 buffersLen := len(buffers)
805 // This is very first buffer so there is no data to append to, prepare empty one
807 buffers = []*txPacketBuffer{{}}
811 lastBuffer := buffers[buffersLen-1]
813 // Last buffer is jumbo buffer, create new buffer
814 if lastBuffer.size > memif.bufferSize {
815 lastBuffer = &txPacketBuffer{}
816 buffers = append(buffers, lastBuffer)
819 // Determine buffer size by max packet size in buffer
820 if packetLen > lastBuffer.size {
821 lastBuffer.size = packetLen
824 lastBuffer.packets = append(lastBuffer.packets, packet)
829 // Reallocate Tx buffers if needed to fit the input packets.
830 log.Debugf("%v - total buffer to allocate count %v", cQueueID, bufCount)
831 pb := &memif.txQueueBufs[queueID]
832 if pb.count < bufCount {
833 newBuffers := C.realloc(unsafe.Pointer(pb.buffers), C.size_t(bufCount*int(C.sizeof_memif_buffer_t)))
834 if newBuffers == nil {
835 // Realloc failed, <count> will be less than len(packets).
838 pb.buffers = (*C.memif_buffer_t)(newBuffers)
843 // Allocate ring slots.
844 var allocated C.uint16_t
845 var subCount C.uint16_t
846 for _, buffer := range buffers {
847 packetCount := C.uint16_t(len(buffer.packets))
848 isJumbo := buffer.size > memif.bufferSize
850 log.Debugf("%v - trying to send max buff size %v, packets len %v, buffer len %v, jumbo %v",
851 cQueueID, buffer.size, len(buffer.packets), packetCount, isJumbo)
853 var nextFreeBuff *C.memif_buffer_t
854 startOffset := allocated
855 errCode := C.govpp_memif_buffer_alloc(memif.cHandle, cQueueID, pb.buffers, startOffset, &nextFreeBuff,
856 packetCount, &allocated, C.uint16_t(buffer.size))
858 err = getMemifError(int(errCode))
859 endEarly := err == ErrNoBufRing
861 // Not enough ring slots, <count> will be less than packetCount.
868 // Copy packet data into the buffers.
869 nowAllocated := allocated - startOffset
870 toFill := nowAllocated
872 // If this is not jumbo frame, only 1 packet needs to be copied each iteration
876 // Iterate over all packets and try to fill them into allocated buffers
877 // If packet is jumbo frame, continue filling to allocated buffers until no buffer is left
878 for i, packet := range buffer.packets {
879 if i >= int(nowAllocated) {
880 // There was less allocated buffers than actual packet count so exit early
884 packetData := unsafe.Pointer(&packet[0])
885 C.govpp_copy_packet_data(nextFreeBuff, toFill, C.int(i), packetData, C.uint16_t(len(packet)))
888 if isJumbo && nowAllocated > 0 {
889 // If we successfully allocated required amount of buffers for entire jumbo to be sent
890 // simply sub entire amount of jumbo frame packets and leave only 1 so sender will think
891 // it only sent 1 packet so it does not need to know anything about jumbo frames
892 subCount += nowAllocated - 1
895 // If we do not have enough buffers left to allocate, simply end here to avoid packet loss and try
896 // to handle it next burst
902 var sentCount C.uint16_t
903 errCode := C.memif_tx_burst(memif.cHandle, cQueueID, pb.buffers, allocated, &sentCount)
904 err = getMemifError(int(errCode))
909 // Prevent negative values
910 realSent := uint16(sentCount) - uint16(subCount)
911 if subCount > sentCount {
915 log.Debugf("%v - sent %v total allocated buffs %v", cQueueID, sentCount, allocated)
919 // RxBurst is used to receive multiple packets in one call from a selected queue.
920 // <count> is the number of packets to receive. The actual number of packets
921 // received may be smaller. <count> effectively limits the maximum number
922 // of packets to receive in one burst (for a flat, predictable memory usage).
923 // The method is non-blocking even if there are no packets to receive.
924 // It is only valid to call this function if memif is in the connected state.
925 // Multiple RxBurst-s can run concurrently provided that each targets a different
927 func (memif *Memif) RxBurst(queueID uint8, count uint16) (packets []RawPacketData, err error) {
928 var recvCount C.uint16_t
929 packets = make([]RawPacketData, 0)
935 if int(queueID) >= len(memif.rxQueueBufs) {
936 return packets, ErrQueueID
939 // Reallocate Rx buffers if needed to fit the output packets.
940 pb := &memif.rxQueueBufs[queueID]
941 bufCount := int(count)
942 if pb.count < bufCount {
943 newBuffers := C.realloc(unsafe.Pointer(pb.buffers), C.size_t(bufCount*int(C.sizeof_memif_buffer_t)))
944 if newBuffers == nil {
945 // Realloc failed, len(<packets>) will be certainly less than <count>.
948 pb.buffers = (*C.memif_buffer_t)(newBuffers)
953 cQueueID := C.uint16_t(queueID)
954 errCode := C.memif_rx_burst(memif.cHandle, cQueueID, pb.buffers, C.uint16_t(bufCount), &recvCount)
955 err = getMemifError(int(errCode))
957 // More packets to read - the user is expected to run RxBurst() until there
958 // are no more packets to receive.
965 chained := len(pb.rxChainBuf) > 0
967 // We had stored data from previous burst because last buffer in previous burst was chained
968 // so we need to continue appending to this data
969 packets = pb.rxChainBuf
973 // Copy packet data into the instances of RawPacketData.
974 for i := 0; i < int(recvCount); i++ {
976 packetData := C.govpp_get_packet_data(pb.buffers, C.int(i), &packetSize)
977 packetBytes := C.GoBytes(packetData, packetSize)
980 // We have chained buffers, so start merging packet data with last read packet
981 prevPacket := packets[len(packets)-1]
982 packets[len(packets)-1] = append(prevPacket, packetBytes...)
984 packets = append(packets, packetBytes)
987 // Mark last buffer as chained based on property on current buffer so next buffers
988 // will try to append data to this one in case we got jumbo frame
989 chained = C.govpp_is_buffer_chained(pb.buffers, C.int(i)) > 0
993 errCode = C.memif_refill_queue(memif.cHandle, cQueueID, recvCount, 0)
995 err = getMemifError(int(errCode))
997 // Throw away packets to avoid duplicities.
1002 // We did not had enough space to process all chained buffers to the end so simply tell
1003 // reader that it should not process any packets here and save them for next burst
1004 // to finish reading the buffer chain
1005 pb.rxChainBuf = packets
1013 // Close removes the memif interface. If the memif is in the connected state,
1014 // the connection is first properly closed.
1015 // Do not access memif after it is closed, let garbage collector to remove it.
1016 func (memif *Memif) Close() error {
1017 log.WithField("ifName", memif.IfName).Debug("Closing the memif interface")
1019 // Delete memif from C-libmemif.
1020 err := getMemifError(int(C.memif_delete(&memif.cHandle)))
1023 // Close memif-global interrupt channel.
1025 // Close file descriptor stopQPollFd.
1026 C.close(C.int(memif.stopQPollFd))
1030 defer context.lock.Unlock()
1031 // Unregister the interface from the context.
1032 delete(context.memifs, memif.ifIndex)
1033 log.WithField("ifName", memif.IfName).Debug("memif interface was closed")
1038 // initQueues allocates resources associated with Rx/Tx queues.
1039 func (memif *Memif) initQueues() error {
1040 // Get Rx/Tx queues count.
1041 details, err := memif.GetDetails()
1046 log.WithFields(logger.Fields{
1047 "ifName": memif.IfName,
1048 "Rx-count": len(details.RxQueues),
1049 "Tx-count": len(details.TxQueues),
1050 }).Debug("Initializing Rx/Tx queues.")
1052 // Initialize interrupt channels.
1054 for i = 0; i < len(details.RxQueues); i++ {
1055 queueIntCh := make(chan struct{}, 1)
1056 memif.queueIntCh = append(memif.queueIntCh, queueIntCh)
1059 // Initialize Rx/Tx packet buffers.
1060 for i = 0; i < len(details.RxQueues); i++ {
1061 memif.rxQueueBufs = append(memif.rxQueueBufs, CPacketBuffers{})
1062 if !memif.IsMaster {
1063 errCode := C.memif_refill_queue(memif.cHandle, C.uint16_t(i), C.uint16_t(memif.ringSize-1), 0)
1064 err = getMemifError(int(errCode))
1066 log.Warn(err.Error())
1070 for i = 0; i < len(details.TxQueues); i++ {
1071 memif.txQueueBufs = append(memif.txQueueBufs, CPacketBuffers{})
1077 // closeQueues deallocates all resources associated with Rx/Tx queues.
1078 func (memif *Memif) closeQueues() {
1079 log.WithFields(logger.Fields{
1080 "ifName": memif.IfName,
1081 "Rx-count": len(memif.rxQueueBufs),
1082 "Tx-count": len(memif.txQueueBufs),
1083 }).Debug("Closing Rx/Tx queues.")
1085 // Close interrupt channels.
1086 for _, ch := range memif.queueIntCh {
1089 memif.queueIntCh = nil
1091 // Deallocate Rx/Tx packet buffers.
1092 for _, pb := range memif.rxQueueBufs {
1093 C.free(unsafe.Pointer(pb.buffers))
1095 memif.rxQueueBufs = nil
1096 for _, pb := range memif.txQueueBufs {
1097 C.free(unsafe.Pointer(pb.buffers))
1099 memif.txQueueBufs = nil
1102 // pollEvents repeatedly polls for a libmemif event.
1104 defer context.wg.Done()
1106 errCode := C.memif_poll_event(C.int(-1))
1107 err := getMemifError(int(errCode))
1108 if err == ErrPollCanceled {
1114 // pollRxQueue repeatedly polls an Rx queue for interrupts.
1115 func pollRxQueue(memif *Memif, queueID uint8) {
1116 defer memif.wg.Done()
1118 log.WithFields(logger.Fields{
1119 "ifName": memif.IfName,
1120 "queue-ID": queueID,
1121 }).Debug("Started queue interrupt polling.")
1124 errCode := C.memif_get_queue_efd(memif.cHandle, C.uint16_t(queueID), &qfd)
1125 err := getMemifError(int(errCode))
1127 log.WithField("err", err).Error("memif_get_queue_efd() failed")
1131 // Create epoll file descriptor.
1132 var event [1]syscall.EpollEvent
1133 epFd, err := syscall.EpollCreate1(0)
1135 log.WithField("err", err).Error("epoll_create1() failed")
1138 defer syscall.Close(epFd)
1140 // Add Rx queue interrupt file descriptor.
1141 event[0].Events = syscall.EPOLLIN
1142 event[0].Fd = int32(qfd)
1143 if err = syscall.EpollCtl(epFd, syscall.EPOLL_CTL_ADD, int(qfd), &event[0]); err != nil {
1144 log.WithField("err", err).Error("epoll_ctl() failed")
1148 // Add file descriptor used to stop this go routine.
1149 event[0].Events = syscall.EPOLLIN
1150 event[0].Fd = int32(memif.stopQPollFd)
1151 if err = syscall.EpollCtl(epFd, syscall.EPOLL_CTL_ADD, memif.stopQPollFd, &event[0]); err != nil {
1152 log.WithField("err", err).Error("epoll_ctl() failed")
1156 // Poll for interrupts.
1158 _, err := syscall.EpollWait(epFd, event[:], -1)
1160 errno, _ := err.(syscall.Errno)
1161 //EINTR and EAGAIN should not be considered as a fatal error, try again
1162 if errno == syscall.EINTR || errno == syscall.EAGAIN {
1165 log.WithField("err", err).Error("epoll_wait() failed")
1169 // Handle Rx Interrupt.
1170 if event[0].Fd == int32(qfd) {
1171 // Consume the interrupt event.
1172 buf := make([]byte, 8)
1173 _, err = syscall.Read(int(qfd), buf[:])
1175 log.WithField("err", err).Warn("read() failed")
1178 // Send signal to memif-global interrupt channel.
1180 case memif.intCh <- queueID:
1186 // Send signal to queue-specific interrupt channel.
1188 case memif.queueIntCh[queueID] <- struct{}{}:
1195 // Stop the go routine if requested.
1196 if event[0].Fd == int32(memif.stopQPollFd) {
1197 log.WithFields(logger.Fields{
1198 "ifName": memif.IfName,
1199 "queue-ID": queueID,
1200 }).Debug("Stopped queue interrupt polling.")
1206 //export go_on_connect_callback
1207 func go_on_connect_callback(privateCtx unsafe.Pointer) C.int {
1208 log.Debug("go_on_connect_callback BEGIN")
1209 defer log.Debug("go_on_connect_callback END")
1210 context.lock.RLock()
1211 defer context.lock.RUnlock()
1213 // Get memif reference.
1214 ifIndex := int(uintptr(privateCtx))
1215 memif, exists := context.memifs[ifIndex]
1217 return C.int(ErrNoConn.Code())
1220 // Initialize Rx/Tx queues.
1221 err := memif.initQueues()
1223 if memifErr, ok := err.(*MemifError); ok {
1224 return C.int(memifErr.Code())
1226 return C.int(ErrUnknown.Code())
1229 // Call the user callback.
1230 if memif.callbacks.OnConnect != nil {
1231 memif.callbacks.OnConnect(memif)
1234 // Start polling the RX queues for interrupts.
1235 for i := 0; i < len(memif.queueIntCh); i++ {
1237 go pollRxQueue(memif, uint8(i))
1243 //export go_on_disconnect_callback
1244 func go_on_disconnect_callback(privateCtx unsafe.Pointer) C.int {
1245 log.Debug("go_on_disconnect_callback BEGIN")
1246 defer log.Debug("go_on_disconnect_callback END")
1247 context.lock.RLock()
1248 defer context.lock.RUnlock()
1250 // Get memif reference.
1251 ifIndex := int(uintptr(privateCtx))
1252 memif, exists := context.memifs[ifIndex]
1258 // Stop polling the RX queues for interrupts.
1259 buf := make([]byte, 8)
1260 binary.PutUvarint(buf, 1)
1262 _, err := syscall.Write(memif.stopQPollFd, buf[:])
1264 return C.int(ErrSyscall.Code())
1268 // - remove the event
1269 _, err = syscall.Read(memif.stopQPollFd, buf[:])
1271 return C.int(ErrSyscall.Code())
1274 // Call the user callback.
1275 if memif.callbacks.OnDisconnect != nil {
1276 memif.callbacks.OnDisconnect(memif)
1279 // Close Rx/Tx queues.