-#if DPDK > 0
-clib_error_t *
-vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
- unsigned socket_id)
-{
- vlib_buffer_main_t *bm = vm->buffer_main;
- vlib_physmem_main_t *vpm = &vm->physmem_main;
- struct rte_mempool *rmp;
- int i;
-
- if (!rte_pktmbuf_pool_create)
- return clib_error_return (0, "not linked with DPDK");
-
- vec_validate_aligned (bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
-
- /* pool already exists, nothing to do */
- if (bm->pktmbuf_pools[socket_id])
- return 0;
-
- u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0);
-
- rmp = rte_pktmbuf_pool_create ((char *) pool_name, /* pool name */
- num_mbufs, /* number of mbufs */
- 512, /* cache size */
- VLIB_BUFFER_HDR_SIZE, /* priv size */
- VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE, /* dataroom size */
- socket_id); /* cpu socket */
-
- if (rmp)
- {
- {
- uword this_pool_end;
- uword this_pool_start;
- uword this_pool_size;
- uword save_vpm_start, save_vpm_end, save_vpm_size;
- struct rte_mempool_memhdr *memhdr;
-
- this_pool_start = ~0ULL;
- this_pool_end = 0LL;
-
- STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
- {
- if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
- this_pool_end = (uword) (memhdr->addr + memhdr->len);
- if (((uword) memhdr->addr) < this_pool_start)
- this_pool_start = (uword) (memhdr->addr);
- }
- ASSERT (this_pool_start < ~0ULL && this_pool_end > 0);
- this_pool_size = this_pool_end - this_pool_start;
-
- if (CLIB_DEBUG > 1)
- {
- clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
- pool_name, this_pool_start, this_pool_end,
- this_pool_size);
- clib_warning
- ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
- vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
- }
-
- save_vpm_start = vpm->virtual.start;
- save_vpm_end = vpm->virtual.end;
- save_vpm_size = vpm->virtual.size;
-
- if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
- vpm->virtual.start = this_pool_start;
- if (this_pool_end > vpm->virtual.end)
- vpm->virtual.end = this_pool_end;
-
- vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
-
- if (CLIB_DEBUG > 1)
- {
- clib_warning
- ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
- vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
- }
-
- /* check if fits into buffer index range */
- if ((u64) vpm->virtual.size >
- ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
- {
- clib_warning ("physmem: virtual size out of range!");
- vpm->virtual.start = save_vpm_start;
- vpm->virtual.end = save_vpm_end;
- vpm->virtual.size = save_vpm_size;
- rmp = 0;
- }
- }
- if (rmp)
- {
- bm->pktmbuf_pools[socket_id] = rmp;
- vec_free (pool_name);
- return 0;
- }
- }
-
- vec_free (pool_name);
-
- /* no usable pool for this socket, try to use pool from another one */
- for (i = 0; i < vec_len (bm->pktmbuf_pools); i++)
- {
- if (bm->pktmbuf_pools[i])
- {
- clib_warning
- ("WARNING: Failed to allocate mempool for CPU socket %u. "
- "Threads running on socket %u will use socket %u mempool.",
- socket_id, socket_id, i);
- bm->pktmbuf_pools[socket_id] = bm->pktmbuf_pools[i];
- return 0;
- }
- }
-
- return clib_error_return (0, "failed to allocate mempool on socket %u",
- socket_id);
-}
-#endif
-
-static void
-vlib_serialize_tx (serialize_main_header_t * m, serialize_stream_t * s)
-{
- vlib_main_t *vm;
- vlib_serialize_buffer_main_t *sm;
- uword n, n_bytes_to_write;
- vlib_buffer_t *last;
-
- n_bytes_to_write = s->current_buffer_index;
- sm =
- uword_to_pointer (s->data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vm = sm->vlib_main;
-
- ASSERT (sm->tx.max_n_data_bytes_per_chain > 0);
- if (serialize_stream_is_end_of_stream (s)
- || sm->tx.n_total_data_bytes + n_bytes_to_write >
- sm->tx.max_n_data_bytes_per_chain)
- {
- vlib_process_t *p = vlib_get_current_process (vm);
-
- last = vlib_get_buffer (vm, sm->last_buffer);
- last->current_length = n_bytes_to_write;
-
- vlib_set_next_frame_buffer (vm, &p->node_runtime, sm->tx.next_index,
- sm->first_buffer);
-
- sm->first_buffer = sm->last_buffer = ~0;
- sm->tx.n_total_data_bytes = 0;
- }
-
- else if (n_bytes_to_write == 0 && s->n_buffer_bytes == 0)
- {
- ASSERT (sm->first_buffer == ~0);
- ASSERT (sm->last_buffer == ~0);
- n =
- vlib_buffer_alloc_from_free_list (vm, &sm->first_buffer, 1,
- sm->tx.free_list_index);
- if (n != 1)
- serialize_error (m,
- clib_error_create
- ("vlib_buffer_alloc_from_free_list fails"));
- sm->last_buffer = sm->first_buffer;
- s->n_buffer_bytes =
- vlib_buffer_free_list_buffer_size (vm, sm->tx.free_list_index);
- }
-
- if (n_bytes_to_write > 0)
- {
- vlib_buffer_t *prev = vlib_get_buffer (vm, sm->last_buffer);
- n =
- vlib_buffer_alloc_from_free_list (vm, &sm->last_buffer, 1,
- sm->tx.free_list_index);
- if (n != 1)
- serialize_error (m,
- clib_error_create
- ("vlib_buffer_alloc_from_free_list fails"));
- sm->tx.n_total_data_bytes += n_bytes_to_write;
- prev->current_length = n_bytes_to_write;
- prev->next_buffer = sm->last_buffer;
- prev->flags |= VLIB_BUFFER_NEXT_PRESENT;
- }
-
- if (sm->last_buffer != ~0)
- {
- last = vlib_get_buffer (vm, sm->last_buffer);
- s->buffer = vlib_buffer_get_current (last);
- s->current_buffer_index = 0;
- ASSERT (last->current_data == s->current_buffer_index);
- }
-}
-
-static void
-vlib_serialize_rx (serialize_main_header_t * m, serialize_stream_t * s)
-{
- vlib_main_t *vm;
- vlib_serialize_buffer_main_t *sm;
- vlib_buffer_t *last;
-
- sm =
- uword_to_pointer (s->data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vm = sm->vlib_main;
-
- if (serialize_stream_is_end_of_stream (s))
- return;
-
- if (sm->last_buffer != ~0)
- {
- last = vlib_get_buffer (vm, sm->last_buffer);
-
- if (last->flags & VLIB_BUFFER_NEXT_PRESENT)
- sm->last_buffer = last->next_buffer;
- else
- {
- vlib_buffer_free (vm, &sm->first_buffer, /* count */ 1);
- sm->first_buffer = sm->last_buffer = ~0;
- }
- }
-
- if (sm->last_buffer == ~0)
- {
- while (clib_fifo_elts (sm->rx.buffer_fifo) == 0)
- {
- sm->rx.ready_one_time_event =
- vlib_process_create_one_time_event (vm, vlib_current_process (vm),
- ~0);
- vlib_process_wait_for_one_time_event (vm, /* no event data */ 0,
- sm->rx.ready_one_time_event);
- }
-
- clib_fifo_sub1 (sm->rx.buffer_fifo, sm->first_buffer);
- sm->last_buffer = sm->first_buffer;
- }
-
- ASSERT (sm->last_buffer != ~0);
-
- last = vlib_get_buffer (vm, sm->last_buffer);
- s->current_buffer_index = 0;
- s->buffer = vlib_buffer_get_current (last);
- s->n_buffer_bytes = last->current_length;
-}
-
-static void
-serialize_open_vlib_helper (serialize_main_t * m,
- vlib_main_t * vm,
- vlib_serialize_buffer_main_t * sm, uword is_read)
-{
- /* Initialize serialize main but save overflow buffer for re-use between calls. */
- {
- u8 *save = m->stream.overflow_buffer;
- memset (m, 0, sizeof (m[0]));
- m->stream.overflow_buffer = save;
- if (save)
- _vec_len (save) = 0;
- }
-
- sm->first_buffer = sm->last_buffer = ~0;
- if (is_read)
- clib_fifo_reset (sm->rx.buffer_fifo);
- else
- sm->tx.n_total_data_bytes = 0;
- sm->vlib_main = vm;
- m->header.data_function = is_read ? vlib_serialize_rx : vlib_serialize_tx;
- m->stream.data_function_opaque = pointer_to_uword (sm);
-}
-
-void
-serialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm,
- vlib_serialize_buffer_main_t * sm)
-{
- serialize_open_vlib_helper (m, vm, sm, /* is_read */ 0);
-}
-
-void
-unserialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm,
- vlib_serialize_buffer_main_t * sm)
-{
- serialize_open_vlib_helper (m, vm, sm, /* is_read */ 1);
-}
-
-u32
-serialize_close_vlib_buffer (serialize_main_t * m)
-{
- vlib_serialize_buffer_main_t *sm
- = uword_to_pointer (m->stream.data_function_opaque,
- vlib_serialize_buffer_main_t *);
- vlib_buffer_t *last;
- serialize_stream_t *s = &m->stream;
-
- last = vlib_get_buffer (sm->vlib_main, sm->last_buffer);
- last->current_length = s->current_buffer_index;
-
- if (vec_len (s->overflow_buffer) > 0)
- {
- sm->last_buffer
- = vlib_buffer_add_data (sm->vlib_main, sm->tx.free_list_index,
- sm->last_buffer == ~0 ? 0 : sm->last_buffer,
- s->overflow_buffer,
- vec_len (s->overflow_buffer));
- _vec_len (s->overflow_buffer) = 0;
- }
-
- return sm->first_buffer;
-}
-
-void
-unserialize_close_vlib_buffer (serialize_main_t * m)
-{
- vlib_serialize_buffer_main_t *sm
- = uword_to_pointer (m->stream.data_function_opaque,
- vlib_serialize_buffer_main_t *);
- if (sm->first_buffer != ~0)
- vlib_buffer_free_one (sm->vlib_main, sm->first_buffer);
- clib_fifo_reset (sm->rx.buffer_fifo);
- if (m->stream.overflow_buffer)
- _vec_len (m->stream.overflow_buffer) = 0;
-}