2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
21 #include <sys/ioctl.h>
24 #include <vlib/vlib.h>
25 #include <vlib/unix/unix.h>
26 #include <vnet/ethernet/ethernet.h>
28 #include <memif/memif.h>
29 #include <memif/private.h>
31 #define foreach_memif_tx_func_error \
32 _(NO_FREE_SLOTS, "no free tx slots") \
33 _(ROLLBACK, "no enough space in tx buffers")
37 #define _(f,s) MEMIF_TX_ERROR_##f,
38 foreach_memif_tx_func_error
41 } memif_tx_func_error_t;
43 static char *memif_tx_func_error_strings[] = {
45 foreach_memif_tx_func_error
49 #ifndef CLIB_MARCH_VARIANT
51 format_memif_device_name (u8 * s, va_list * args)
53 u32 dev_instance = va_arg (*args, u32);
54 memif_main_t *mm = &memif_main;
55 memif_if_t *mif = pool_elt_at_index (mm->interfaces, dev_instance);
56 memif_socket_file_t *msf;
58 msf = pool_elt_at_index (mm->socket_files, mif->socket_file_index);
59 s = format (s, "memif%lu/%lu", msf->socket_id, mif->id);
65 format_memif_device (u8 * s, va_list * args)
67 u32 dev_instance = va_arg (*args, u32);
68 int verbose = va_arg (*args, int);
69 u32 indent = format_get_indent (s);
71 s = format (s, "MEMIF interface");
74 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
81 format_memif_tx_trace (u8 * s, va_list * args)
83 s = format (s, "Unimplemented...");
87 static_always_inline void
88 memif_add_copy_op (memif_per_thread_data_t * ptd, void *data, u32 len,
89 u16 buffer_offset, u16 buffer_vec_index)
92 vec_add2_aligned (ptd->copy_ops, co, 1, CLIB_CACHE_LINE_BYTES);
95 co->buffer_offset = buffer_offset;
96 co->buffer_vec_index = buffer_vec_index;
99 static_always_inline uword
100 memif_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
101 vlib_frame_t * frame, memif_if_t * mif,
102 memif_ring_type_t type, memif_queue_t * mq,
103 memif_per_thread_data_t * ptd)
106 u32 *buffers = vlib_frame_vector_args (frame);
107 u32 n_left = frame->n_vectors;
109 u16 ring_size, mask, slot, free_slots;
111 vlib_buffer_t *b0, *b1, *b2, *b3;
113 memif_region_index_t last_region = ~0;
114 void *last_region_shm = 0;
117 ring_size = 1 << mq->log2_ring_size;
118 mask = ring_size - 1;
122 free_slots = ring->tail - mq->last_tail;
123 mq->last_tail += free_slots;
124 slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
126 if (type == MEMIF_RING_S2M)
127 free_slots = ring_size - ring->head + mq->last_tail;
129 free_slots = ring->head - ring->tail;
131 while (n_left && free_slots)
136 u32 bi0, dst_off, src_left, dst_left, bytes_to_copy;
137 u32 saved_ptd_copy_ops_len = _vec_len (ptd->copy_ops);
138 u32 saved_ptd_buffers_len = _vec_len (ptd->buffers);
139 u16 saved_slot = slot;
141 CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
144 d0 = &ring->desc[slot & mask];
145 if (PREDICT_FALSE (last_region != d0->region))
147 last_region_shm = mif->regions[d0->region].shm;
148 last_region = d0->region;
150 mb0 = last_region_shm + d0->offset;
154 /* slave is the producer, so it should be able to reset buffer length */
155 dst_left = (type == MEMIF_RING_S2M) ? mif->run.buffer_size : d0->length;
157 if (PREDICT_TRUE (n_left >= 4))
158 vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD);
163 b0 = vlib_get_buffer (vm, bi0);
164 src_off = b0->current_data;
165 src_left = b0->current_length;
169 if (PREDICT_FALSE (dst_left == 0))
175 d0->flags = MEMIF_DESC_FLAG_NEXT;
176 d0 = &ring->desc[slot & mask];
180 MEMIF_RING_S2M) ? mif->run.buffer_size : d0->length;
182 if (PREDICT_FALSE (last_region != d0->region))
184 last_region_shm = mif->regions[d0->region].shm;
185 last_region = d0->region;
187 mb0 = last_region_shm + d0->offset;
191 /* we need to rollback vectors before bailing out */
192 _vec_len (ptd->buffers) = saved_ptd_buffers_len;
193 _vec_len (ptd->copy_ops) = saved_ptd_copy_ops_len;
194 vlib_error_count (vm, node->node_index,
195 MEMIF_TX_ERROR_ROLLBACK, 1);
200 bytes_to_copy = clib_min (src_left, dst_left);
201 memif_add_copy_op (ptd, mb0 + dst_off, bytes_to_copy, src_off,
202 vec_len (ptd->buffers));
203 vec_add1_aligned (ptd->buffers, bi0, CLIB_CACHE_LINE_BYTES);
204 src_off += bytes_to_copy;
205 dst_off += bytes_to_copy;
206 src_left -= bytes_to_copy;
207 dst_left -= bytes_to_copy;
210 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
212 bi0 = b0->next_buffer;
216 d0->length = dst_off;
228 n_copy_op = vec_len (ptd->copy_ops);
230 while (n_copy_op >= 8)
232 CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD);
233 CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD);
234 CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD);
235 CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD);
237 b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
238 b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]);
239 b2 = vlib_get_buffer (vm, ptd->buffers[co[2].buffer_vec_index]);
240 b3 = vlib_get_buffer (vm, ptd->buffers[co[3].buffer_vec_index]);
242 clib_memcpy_fast (co[0].data, b0->data + co[0].buffer_offset,
244 clib_memcpy_fast (co[1].data, b1->data + co[1].buffer_offset,
246 clib_memcpy_fast (co[2].data, b2->data + co[2].buffer_offset,
248 clib_memcpy_fast (co[3].data, b3->data + co[3].buffer_offset,
256 b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]);
257 clib_memcpy_fast (co[0].data, b0->data + co[0].buffer_offset,
263 vec_reset_length (ptd->copy_ops);
264 vec_reset_length (ptd->buffers);
266 CLIB_MEMORY_STORE_BARRIER ();
267 if (type == MEMIF_RING_S2M)
272 if (n_left && n_retries--)
275 clib_spinlock_unlock_if_init (&mif->lockp);
279 vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
283 if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
286 CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
290 vlib_buffer_free (vm, vlib_frame_vector_args (frame), frame->n_vectors);
292 return frame->n_vectors;
295 static_always_inline uword
296 memif_interface_tx_zc_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
297 vlib_frame_t * frame, memif_if_t * mif,
299 memif_per_thread_data_t * ptd)
301 memif_ring_t *ring = mq->ring;
302 u32 *buffers = vlib_frame_vector_args (frame);
303 u32 n_left = frame->n_vectors;
304 u16 slot, free_slots, n_free;
305 u16 ring_size = 1 << mq->log2_ring_size;
306 u16 mask = ring_size - 1;
311 n_free = ring->tail - mq->last_tail;
314 vlib_buffer_free_from_ring_no_next (vm, mq->buffers,
315 mq->last_tail & mask,
317 mq->last_tail += n_free;
321 free_slots = ring_size - ring->head + mq->last_tail;
323 while (n_left && free_slots)
326 u16 slots_in_packet = 1;
330 CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES,
333 if (PREDICT_TRUE (n_left >= 4))
334 vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD);
340 d0 = &ring->desc[s0];
341 mq->buffers[s0] = bi0;
342 b0 = vlib_get_buffer (vm, bi0);
344 d0->region = b0->buffer_pool_index + 1;
345 d0->offset = (void *) b0->data + b0->current_data -
346 mif->regions[d0->region].shm;
347 d0->length = b0->current_length;
352 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
354 if (PREDICT_FALSE (free_slots == 0))
356 /* revert to last fully processed packet */
357 free_slots += slots_in_packet;
358 slot -= slots_in_packet;
362 d0->flags = MEMIF_DESC_FLAG_NEXT;
363 bi0 = b0->next_buffer;
378 CLIB_MEMORY_STORE_BARRIER ();
381 if (n_left && n_retries--)
384 clib_spinlock_unlock_if_init (&mif->lockp);
388 vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
390 vlib_buffer_free (vm, buffers, n_left);
393 if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0 && mq->int_fd > -1)
396 CLIB_UNUSED (int r) = write (mq->int_fd, &b, sizeof (b));
400 return frame->n_vectors;
403 VNET_DEVICE_CLASS_TX_FN (memif_device_class) (vlib_main_t * vm,
404 vlib_node_runtime_t * node,
405 vlib_frame_t * frame)
407 memif_main_t *nm = &memif_main;
408 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
409 memif_if_t *mif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
411 u32 thread_index = vm->thread_index;
412 memif_per_thread_data_t *ptd = vec_elt_at_index (memif_main.per_thread_data,
414 u8 tx_queues = vec_len (mif->tx_queues);
416 if (tx_queues < vec_len (vlib_mains))
418 ASSERT (tx_queues > 0);
419 mq = vec_elt_at_index (mif->tx_queues, thread_index % tx_queues);
420 clib_spinlock_lock_if_init (&mif->lockp);
423 mq = vec_elt_at_index (mif->tx_queues, thread_index);
425 if (mif->flags & MEMIF_IF_FLAG_ZERO_COPY)
426 return memif_interface_tx_zc_inline (vm, node, frame, mif, mq, ptd);
427 else if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
428 return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_S2M,
431 return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_M2S,
436 memif_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
439 memif_main_t *apm = &memif_main;
440 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
441 memif_if_t *mif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
443 /* Shut off redirection */
444 if (node_index == ~0)
446 mif->per_interface_next_index = node_index;
450 mif->per_interface_next_index =
451 vlib_node_add_next (vlib_get_main (), memif_input_node.index, node_index);
455 memif_clear_hw_interface_counters (u32 instance)
457 /* Nothing for now */
460 static clib_error_t *
461 memif_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
462 vnet_hw_interface_rx_mode mode)
464 memif_main_t *mm = &memif_main;
465 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
466 memif_if_t *mif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
467 memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, qid);
469 if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING)
470 mq->ring->flags |= MEMIF_RING_FLAG_MASK_INT;
472 mq->ring->flags &= ~MEMIF_RING_FLAG_MASK_INT;
477 static clib_error_t *
478 memif_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
480 memif_main_t *mm = &memif_main;
481 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
482 memif_if_t *mif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
483 static clib_error_t *error = 0;
485 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
486 mif->flags |= MEMIF_IF_FLAG_ADMIN_UP;
488 mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP;
493 static clib_error_t *
494 memif_subif_add_del_function (vnet_main_t * vnm,
496 struct vnet_sw_interface_t *st, int is_add)
498 /* Nothing for now */
503 VNET_DEVICE_CLASS (memif_device_class) = {
505 .format_device_name = format_memif_device_name,
506 .format_device = format_memif_device,
507 .format_tx_trace = format_memif_tx_trace,
508 .tx_function_n_errors = MEMIF_TX_N_ERROR,
509 .tx_function_error_strings = memif_tx_func_error_strings,
510 .rx_redirect_to_node = memif_set_interface_next_node,
511 .clear_counters = memif_clear_hw_interface_counters,
512 .admin_up_down_function = memif_interface_admin_up_down,
513 .subif_add_del_function = memif_subif_add_del_function,
514 .rx_mode_change_function = memif_interface_rx_mode_change,
520 * fd.io coding-style-patch-verification: ON
523 * eval: (c-set-style "gnu")