svm_fifo_t *f = s->data.tx_fifo;
rv = clib_min (svm_fifo_max_enqueue_prod (f), bytes_this_chunk);
svm_fifo_enqueue_nocopy (f, rv);
- session_send_io_evt_to_thread_custom (&f->master_session_index,
- s->thread_index,
- SESSION_IO_EVT_TX);
+ session_send_io_evt_to_thread_custom (
+ &f->shr->master_session_index, s->thread_index, SESSION_IO_EVT_TX);
}
else
rv = app_send_stream (&s->data, test_data + test_buf_offset,
hdr.lcl_port = at->lcl_port;
svm_fifo_enqueue (f, sizeof (hdr), (u8 *) & hdr);
svm_fifo_enqueue_nocopy (f, rv);
- session_send_io_evt_to_thread_custom (&f->master_session_index,
- s->thread_index,
- SESSION_IO_EVT_TX);
+ session_send_io_evt_to_thread_custom (
+ &f->shr->master_session_index, s->thread_index, SESSION_IO_EVT_TX);
}
else
{
session->bytes_to_send = ecm->bytes_to_send;
session->bytes_to_receive = ecm->no_return ? 0ULL : ecm->bytes_to_send;
session->data.rx_fifo = s->rx_fifo;
- session->data.rx_fifo->client_session_index = session_index;
+ session->data.rx_fifo->shr->client_session_index = session_index;
session->data.tx_fifo = s->tx_fifo;
- session->data.tx_fifo->client_session_index = session_index;
+ session->data.tx_fifo->shr->client_session_index = session_index;
session->data.vpp_evt_q = ecm->vpp_event_queue[thread_index];
session->vpp_session_handle = session_handle (s);
session->bytes_to_send = ecm->bytes_to_send;
session->bytes_to_receive = ecm->no_return ? 0ULL : ecm->bytes_to_send;
session->data.rx_fifo = s->rx_fifo;
- session->data.rx_fifo->client_session_index = session_index;
+ session->data.rx_fifo->shr->client_session_index = session_index;
session->data.tx_fifo = s->tx_fifo;
- session->data.tx_fifo->client_session_index = session_index;
+ session->data.tx_fifo->shr->client_session_index = session_index;
session->data.vpp_evt_q = ecm->vpp_event_queue[thread_index];
session->vpp_session_handle = session_handle (s);
return -1;
}
- sp = pool_elt_at_index (ecm->sessions, s->rx_fifo->client_session_index);
+ sp =
+ pool_elt_at_index (ecm->sessions, s->rx_fifo->shr->client_session_index);
receive_data_chunk (ecm, sp);
if (svm_fifo_max_dequeue_cons (s->rx_fifo))
if (svm_fifo_set_event (ao_tx_fifo))
{
u32 ao_thread_index = ao_tx_fifo->master_thread_index;
- u32 ao_session_index = ao_tx_fifo->master_session_index;
+ u32 ao_session_index = ao_tx_fifo->shr->master_session_index;
if (session_send_io_evt_to_thread_custom (&ao_session_index,
ao_thread_index,
SESSION_IO_EVT_TX))
* Reset the active-open tx-fifo master indices so the active-open session
* will receive data, etc.
*/
- s->tx_fifo->master_session_index = s->session_index;
+ s->tx_fifo->shr->master_session_index = s->session_index;
s->tx_fifo->master_thread_index = s->thread_index;
/*
if (svm_fifo_set_event (proxy_tx_fifo))
{
u8 thread_index = proxy_tx_fifo->master_thread_index;
- u32 session_index = proxy_tx_fifo->master_session_index;
+ u32 session_index = proxy_tx_fifo->shr->master_session_index;
return session_send_io_evt_to_thread_custom (&session_index,
thread_index,
SESSION_IO_EVT_TX);
int rv;
if (!svm_fifo_set_event (s->rx_fifo))
return;
- if ((rv =
- app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo->master_session_index,
- SESSION_IO_EVT_RX, SVM_Q_WAIT)))
+ if ((rv = app_send_io_evt_to_vpp (s->vpp_evt_q,
+ s->rx_fifo->shr->master_session_index,
+ SESSION_IO_EVT_RX, SVM_Q_WAIT)))
ECHO_FAIL (ECHO_FAIL_SEND_IO_EVT, "app_send_io_evt_to_vpp errored %d",
rv);
svm_fifo_clear_deq_ntf (s->rx_fifo);
{
app_session_evt_t _app_evt, *app_evt = &_app_evt;
session_accepted_reply_msg_t *rmp;
- svm_fifo_t *rx_fifo, *tx_fifo;
echo_main_t *em = &echo_main;
echo_session_t *session, *ls;
"Unknown listener handle 0x%lx", mp->listener_handle);
return;
}
- if (echo_segment_lookup (mp->segment_handle) == ~0)
+
+ /* Allocate local session and set it up */
+ session = echo_session_new (em);
+
+ if (echo_attach_session (mp->segment_handle, mp->server_rx_fifo,
+ mp->server_tx_fifo, session))
{
ECHO_FAIL (ECHO_FAIL_ACCEPTED_WAIT_FOR_SEG_ALLOC,
"accepted wait_for_segment_allocation errored");
return;
}
- /* Allocate local session and set it up */
- session = echo_session_new (em);
- session->vpp_session_handle = mp->handle;
-
- rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
- rx_fifo->client_session_index = session->session_index;
- tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
- tx_fifo->client_session_index = session->session_index;
+ session->vpp_evt_q =
+ uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *);
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
+ session->vpp_session_handle = mp->handle;
/* session->transport needed by app_send_dgram */
clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip,
session->transport.lcl_port = em->uri_elts.port;
session->vpp_session_handle = mp->handle;
- session->start = clib_time_now (&em->clib_time);
- session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
- svm_msg_q_t *);
session->listener_index = ls->session_index;
+ session->start = clib_time_now (&em->clib_time);
/* Add it to lookup table */
ECHO_LOG (2, "Accepted session 0x%lx S[%u] -> 0x%lx S[%u]",
echo_main_t *em = &echo_main;
echo_session_t *session;
u32 listener_index = htonl (mp->context);
- svm_fifo_t *rx_fifo, *tx_fifo;
clib_atomic_add_fetch (&em->max_sim_connects, 1);
}
session = echo_session_new (em);
- if (echo_segment_lookup (mp->segment_handle) == ~0)
+
+ if (echo_attach_session (mp->segment_handle, mp->server_rx_fifo,
+ mp->server_tx_fifo, session))
{
ECHO_FAIL (ECHO_FAIL_CONNECTED_WAIT_FOR_SEG_ALLOC,
"connected wait_for_segment_allocation errored");
return;
}
-
- rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
- rx_fifo->client_session_index = session->session_index;
- tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
- tx_fifo->client_session_index = session->session_index;
-
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
- session->vpp_session_handle = mp->handle;
- session->start = clib_time_now (&em->clib_time);
session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
svm_msg_q_t *);
+ session->vpp_session_handle = mp->handle;
+ session->start = clib_time_now (&em->clib_time);
session->listener_index = listener_index;
/* session->transport needed by app_send_dgram */
clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
clib_spinlock_unlock (&em->segment_handles_lock);
}
+int
+echo_attach_session (uword segment_handle, uword rxf_offset, uword txf_offset,
+ echo_session_t *s)
+{
+ svm_fifo_shared_t *rx_fifo, *tx_fifo;
+ echo_main_t *em = &echo_main;
+ fifo_segment_t *fs;
+ u32 fs_index;
+
+ fs_index = echo_segment_lookup (segment_handle);
+ if (fs_index == (u32) ~0)
+ {
+ ECHO_LOG (0, "ERROR: segment for session %u is not mounted!",
+ s->session_index);
+ return -1;
+ }
+
+ rx_fifo = uword_to_pointer (rxf_offset, svm_fifo_shared_t *);
+ tx_fifo = uword_to_pointer (txf_offset, svm_fifo_shared_t *);
+ rx_fifo->client_session_index = s->session_index;
+ tx_fifo->client_session_index = s->session_index;
+
+ clib_spinlock_lock (&em->segment_handles_lock);
+
+ fs = fifo_segment_get_segment (&em->segment_main, fs_index);
+ s->rx_fifo = fifo_segment_alloc_fifo_w_shared (fs, rx_fifo);
+ s->tx_fifo = fifo_segment_alloc_fifo_w_shared (fs, tx_fifo);
+
+ clib_spinlock_unlock (&em->segment_handles_lock);
+
+ return 0;
+}
+
/*
*
* Binary API callbacks
ssvm_segment_type_t type, int fd);
u32 echo_segment_lookup (u64 segment_handle);
void echo_segment_detach (u64 segment_handle);
+int echo_attach_session (uword segment_handle, uword rxf_offset,
+ uword txf_offset, echo_session_t *s);
/* Binary API */
static void
udp_echo_bound_uri_cb (session_bound_msg_t * mp, echo_session_t * session)
{
- svm_fifo_t *rx_fifo, *tx_fifo;
echo_main_t *em = &echo_main;
u32 session_index = session->session_index;
if (!em->i_am_master || em->uri_elts.transport_proto != TRANSPORT_PROTO_UDP)
return;
- rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *);
- tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *);
- rx_fifo->client_session_index = session_index;
- tx_fifo->client_session_index = session_index;
+ if (echo_attach_session (mp->segment_handle, mp->rx_fifo, mp->tx_fifo,
+ session))
+ {
+ ECHO_FAIL (ECHO_FAIL_ACCEPTED_WAIT_FOR_SEG_ALLOC,
+ "accepted wait_for_segment_allocation errored");
+ return;
+ }
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
+ session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
session->transport.is_ip4 = mp->lcl_is_ip4;
clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip,
sizeof (ip46_address_t));
session->transport.lcl_port = mp->lcl_port;
- session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
echo_notify_event (em, ECHO_EVT_FIRST_QCONNECT);
session->session_type = ECHO_SESSION_TYPE_STREAM;
"fifo_segment_get_mem_status %s", states_str[rv]);
/* (virtual) fifo size is still large as it is not updated */
- SEG_MGR_TEST ((rx_fifo->size == most_grown), "rx_fifo->size %u",
- rx_fifo->size);
+ SEG_MGR_TEST ((rx_fifo->shr->size == most_grown), "rx_fifo->size %u",
+ rx_fifo->shr->size);
vnet_app_detach_args_t detach_args = {
.app_index = attach_args.app_index,
s.rx_fifo = rx_fifo;
s.tx_fifo = tx_fifo;
s.session_state = SESSION_STATE_READY;
- counter = (u64 *) rx_fifo->head_chunk->data;
+ counter = (u64 *) rx_fifo->shr->head_chunk->data;
start = vlib_time_now (vm);
pid = fork ();
*/
rv = svm_fifo_enqueue (f, sizeof (u32), (u8 *) test_data);
SFIFO_TEST ((rv == sizeof (u32)), "enqueued %d", rv);
- SFIFO_TEST ((f->tail == 4), "fifo tail %u", f->tail);
+ SFIFO_TEST ((f->shr->tail == 4), "fifo tail %u", f->shr->tail);
/*
* Create 3 chunks in the future. The offsets are relative
*/
for (i = 0; i < 3; i++)
{
- offset = (2 * i + 1) * sizeof (u32) - f->tail;
+ offset = (2 * i + 1) * sizeof (u32) - f->shr->tail;
data = (u8 *) (test_data + (2 * i + 1));
if (i == 0)
{
if (verbose)
vlib_cli_output (vm, "fifo after odd segs: %U", format_svm_fifo, f, 1);
- SFIFO_TEST ((f->tail == 8), "fifo tail %u", f->tail);
+ SFIFO_TEST ((f->shr->tail == 8), "fifo tail %u", f->shr->tail);
SFIFO_TEST ((svm_fifo_n_ooo_segments (f) == 2),
"number of ooo segments %u", svm_fifo_n_ooo_segments (f));
/*
* Try adding a completely overlapped segment
*/
- offset = 3 * sizeof (u32) - f->tail;
+ offset = 3 * sizeof (u32) - f->shr->tail;
data = (u8 *) (test_data + 3);
rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data);
if (rv)
*/
for (i = 3; i > 1; i--)
{
- offset = (2 * i + 0) * sizeof (u32) - f->tail;
+ offset = (2 * i + 0) * sizeof (u32) - f->shr->tail;
data = (u8 *) (test_data + (2 * i + 0));
rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data);
if (verbose)
for (i = 0; i < 4; i++)
{
- offset = (2 * i + 1) * sizeof (u32) - f->tail;
+ offset = (2 * i + 1) * sizeof (u32) - f->shr->tail;
data = (u8 *) (test_data + (2 * i + 1));
rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data);
if (verbose)
}
}
- rv = svm_fifo_enqueue_with_offset (f, 8 - f->tail, 21, data);
+ rv = svm_fifo_enqueue_with_offset (f, 8 - f->shr->tail, 21, data);
SFIFO_TEST ((rv == 0), "ooo enqueued %u", rv);
SFIFO_TEST ((svm_fifo_n_ooo_segments (f) == 1),
"number of ooo segments %u", svm_fifo_n_ooo_segments (f));
for (i = 0; i < 4; i++)
{
- offset = (2 * i + 1) * sizeof (u32) - f->tail;
+ offset = (2 * i + 1) * sizeof (u32) - f->shr->tail;
data = (u8 *) (test_data + (2 * i + 1));
rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data);
if (verbose)
{
tp = vp + i;
data64 = tp->offset;
- svm_fifo_enqueue_with_offset (f, tp->offset - f->tail, tp->len,
- (u8 *) & data64);
+ svm_fifo_enqueue_with_offset (f, tp->offset - f->shr->tail, tp->len,
+ (u8 *) &data64);
}
/* Expected result: one big fat chunk at offset 4 */
{
tp = &test_data[i];
data64 = tp->offset;
- rv = svm_fifo_enqueue_with_offset (f, tp->offset - f->tail, tp->len,
- (u8 *) & data64);
+ rv = svm_fifo_enqueue_with_offset (f, tp->offset - f->shr->tail, tp->len,
+ (u8 *) &data64);
if (rv)
{
clib_warning ("enqueue returned %d", rv);
for (i = !randomize; i < vec_len (generate); i++)
{
tp = generate + i;
- svm_fifo_enqueue_with_offset (f,
- fifo_initial_offset + tp->offset -
- f->tail, tp->len,
- (u8 *) data_pattern + tp->offset);
+ svm_fifo_enqueue_with_offset (
+ f, fifo_initial_offset + tp->offset - f->shr->tail, tp->len,
+ (u8 *) data_pattern + tp->offset);
}
/* Add the first segment in order for non random data */
for (i = test_n_bytes - 1; i > 0; i--)
{
- rv = svm_fifo_enqueue_with_offset (f, fifo_initial_offset + i - f->tail,
- sizeof (u8), &test_data[i]);
+ rv = svm_fifo_enqueue_with_offset (
+ f, fifo_initial_offset + i - f->shr->tail, sizeof (u8), &test_data[i]);
if (verbose)
vlib_cli_output (vm, "add [%d] [%d, %d]", i, i, i + sizeof (u8));
if (rv)
if (compare_data (data_buf, test_data, 0, n_test_bytes, (u32 *) & j))
SFIFO_TEST (0, "[%d] dequeued %u expected %u", j, data_buf[j],
test_data[j]);
- svm_fifo_init_pointers (f, (~0 - i) % f->size, (~0 - i) % f->size);
+ svm_fifo_init_pointers (f, (~0 - i) % f->shr->size,
+ (~0 - i) % f->shr->size);
}
SFIFO_TEST (1, "passed multiple ooo enqueue/dequeue");
SFIFO_TEST (rv == 2, "should have 2 chunks has %u", rv);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- c = f->head_chunk;
+ c = f->shr->head_chunk;
SFIFO_TEST (c->start_byte == 0, "head start byte should be %u", 0);
SFIFO_TEST (c->length == 4096, "head chunk length should be %u", 4096);
- SFIFO_TEST (f->tail_chunk == 0, "no tail chunk");
+ SFIFO_TEST (f->shr->tail_chunk == 0, "no tail chunk");
SFIFO_TEST (f->ooo_enq == 0, "should have no ooo enq chunk");
SFIFO_TEST (f->ooo_deq == 0, "should have no ooo deq chunk");
- c = f->end_chunk;
+ c = f->shr->end_chunk;
SFIFO_TEST (c->start_byte == last_start_byte, "end chunk start byte should"
" be %u", last_start_byte);
SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096);
vlib_cli_output (vm, "[%d] dequeued %u expected %u", i, data_buf[i],
test_data[i]);
SFIFO_TEST ((rv == 0), "dequeued compared to original returned %d", rv);
- SFIFO_TEST (f->head_chunk == 0, "head chunk should be 0");
- SFIFO_TEST (f->tail_chunk == 0, "tail chunk should be 0");
+ SFIFO_TEST (f->shr->head_chunk == 0, "head chunk should be 0");
+ SFIFO_TEST (f->shr->tail_chunk == 0, "tail chunk should be 0");
SFIFO_TEST (f->ooo_deq == 0, "should have no ooo deq chunk");
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- SFIFO_TEST (f->head_chunk == 0, "should have no head chunk");
+ SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk");
/* When new fifo chunks are allocated, tail is initialized */
- SFIFO_TEST (f->tail_chunk != 0, "should have no tail chunk");
+ SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk");
SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk");
- c = f->end_chunk;
+ c = f->shr->end_chunk;
SFIFO_TEST (c->start_byte == last_start_byte,
"end chunk should start at %u", last_start_byte);
SFIFO_TEST (c->length == 8192, "end chunk length should be %u", 8192);
SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- SFIFO_TEST (f->head_chunk == 0, "should have no head chunk");
+ SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk");
/* Fifo is full so tail and ooo_enq should be 0 */
- SFIFO_TEST (f->tail_chunk == 0, "should have no tail chunk");
+ SFIFO_TEST (f->shr->tail_chunk == 0, "should have no tail chunk");
SFIFO_TEST (f->ooo_enq == 0, "should have no ooo enq chunk");
/*
last_start_byte += 8192;
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- SFIFO_TEST (f->head_chunk == 0, "should have no head chunk");
- SFIFO_TEST (f->tail_chunk == 0, "should have no tail chunk");
+ SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk");
+ SFIFO_TEST (f->shr->tail_chunk == 0, "should have no tail chunk");
/* We don't remove the last chunk even when the fifo goes empty */
rv = svm_fifo_n_chunks (f);
SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- SFIFO_TEST (f->head_chunk == 0, "should have no head chunk");
+ SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk");
/* When new fifo chunks are allocated, tail is initialized */
- SFIFO_TEST (f->tail_chunk != 0, "should have no tail chunk");
+ SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk");
SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk");
- c = f->end_chunk;
+ c = f->shr->end_chunk;
SFIFO_TEST (c->start_byte == last_start_byte,
"end chunk should start at %u", last_start_byte);
SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
last_start_byte += 16384;
- c = f->end_chunk;
+ c = f->shr->end_chunk;
SFIFO_TEST (c->start_byte == last_start_byte,
"end chunk should start at %u", last_start_byte);
SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
/* fifo does not end on chunk boundary because of the - 100 */
- SFIFO_TEST (f->head_chunk != 0, "should have head chunk");
- SFIFO_TEST (f->tail_chunk != 0, "should have tail chunk");
+ SFIFO_TEST (f->shr->head_chunk != 0, "should have head chunk");
+ SFIFO_TEST (f->shr->tail_chunk != 0, "should have tail chunk");
/*
* Enqueue and dequeue byte-by-byte ooo
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
last_start_byte += 4096;
- c = f->end_chunk;
+ c = f->shr->end_chunk;
SFIFO_TEST (c->start_byte == last_start_byte,
"end chunk should start at %u", last_start_byte);
SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384);
SFIFO_TEST ((rv == fifo_size), "all bytes should be dropped %u", rv);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
- SFIFO_TEST (f->head_chunk != 0, "should have head chunk");
- SFIFO_TEST (f->tail_chunk != 0, "should have tail chunk");
+ SFIFO_TEST (f->shr->head_chunk != 0, "should have head chunk");
+ SFIFO_TEST (f->shr->tail_chunk != 0, "should have tail chunk");
/* We don't remove the last chunk even when the fifo goes empty */
rv = svm_fifo_n_chunks (f);
svm_fifo_set_size (f, fifo_size);
validate_test_and_buf_vecs (&test_data, &data_buf, fifo_size);
- c = f->start_chunk;
+ c = f->shr->start_chunk;
SFIFO_TEST (c->next == 0, "no next");
svm_fifo_fill_chunk_list (f);
SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
c = svm_fifo_tail_chunk (f);
- SFIFO_TEST (c == f->end_chunk, "tail is end chunk");
+ SFIFO_TEST (c == f->shr->end_chunk, "tail is end chunk");
/* Initialize head chunk */
rv = svm_fifo_max_read_chunk (f);
SFIFO_TEST (rv == 4096, "dequeue should work");
c = svm_fifo_head_chunk (f);
- SFIFO_TEST (c == f->end_chunk, "head chunk should be last");
+ SFIFO_TEST (c == f->shr->end_chunk, "head chunk should be last");
rv = svm_fifo_max_read_chunk (f);
SFIFO_TEST (rv == 0, "max read chunk %u", rv);
svm_fifo_dequeue (f, vec_len (retrieved_data), retrieved_data);
if (memcmp (retrieved_data, test_data, vec_len (retrieved_data)))
{
- result = (u32 *) f->head_chunk->data;
+ result = (u32 *) f->shr->head_chunk->data;
*result = 1;
_exit (0);
}
}
- result = (u32 *) f->head_chunk->data;
+ result = (u32 *) f->shr->head_chunk->data;
*result = 0;
vec_free (test_data);
usleep (1e3);
- result = (u32 *) f->head_chunk->data;
+ result = (u32 *) f->shr->head_chunk->data;
SFIFO_TEST (*result == 0, "slave reported no error");
vec_free (a->new_segment_indices);
rv = fifo_segment_num_free_fifos (fs);
SFIFO_TEST (rv == 50, "prealloc fifo hdrs expected %u is %u", 50, rv);
rv = fifo_segment_free_bytes (fs);
- free_space -= sizeof (svm_fifo_t) * 50;
+ free_space -= sizeof (svm_fifo_shared_t) * 50;
/* Memory alloc alignment accounts for the difference */
SFIFO_TEST (approx_leq (free_space, rv, 128), "free space expected %u is %u",
free_space, rv);
*/
#include <svm/fifo_segment.h>
+#include <vppinfra/mem.h>
static inline void *
fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align)
int
fifo_segment_init (fifo_segment_t * fs)
{
- u32 align = 8, offset = 2 * 4096, slices_sz;
+ u32 align = 8, offset = 2 * 4096, slices_sz, i;
uword max_fifo, seg_start, seg_sz;
fifo_segment_header_t *fsh;
ssvm_shared_header_t *sh;
fs->max_byte_index = fsh->max_byte_index;
fs->h = sh->opaque[0] = fsh;
+ vec_validate (fs->slices, fs->n_slices - 1);
+ for (i = 0; i < fs->n_slices; i++)
+ fs->slices[i].fifos =
+ clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32);
+
sh->ready = 1;
return (0);
}
goto done;
fs->max_byte_index = fsh->max_byte_index;
+ vec_validate (fs->slices, 0);
+ fs->slices[0].fifos =
+ clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32);
done:
vec_add1 (a->new_segment_indices, fs - sm->segments);
void
fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s)
{
+ fifo_segment_cleanup (s);
ssvm_delete (&s->ssvm);
clib_memset (s, 0xfe, sizeof (*s));
pool_put (sm->segments, s);
fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
fifo_segment_slice_t * fss, u32 batch_size)
{
- svm_fifo_t *f;
+ svm_fifo_shared_t *f;
uword size;
u8 *fmem;
int i;
/* Carve fifo hdr space */
for (i = 0; i < batch_size; i++)
{
- f = (svm_fifo_t *) fmem;
+ f = (svm_fifo_shared_t *) fmem;
memset (f, 0, sizeof (*f));
f->next = fss->free_fifos;
fss->free_fifos = f;
return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size);
}
-static svm_fifo_t *
-fsh_try_alloc_fifo_hdr (fifo_segment_header_t * fsh,
- fifo_segment_slice_t * fss)
+static svm_fifo_shared_t *
+fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
{
- svm_fifo_t *f;
+ svm_fifo_shared_t *f;
if (!fss->free_fifos)
{
* - single fifo allocation
* - grab multiple fifo chunks from freelists
*/
-static svm_fifo_t *
-fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
- u32 data_bytes)
+static svm_fifo_shared_t *
+fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
{
+ fifo_segment_slice_t *fss;
u32 fl_index, min_size;
svm_fifo_chunk_t *c;
- svm_fifo_t *f = 0;
+ svm_fifo_shared_t *sf = 0;
+ fss = fsh_slice_get (fsh, slice_index);
min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
fl_index = fs_freelist_for_size (min_size);
if (!fss_chunk_fl_index_is_valid (fss, fl_index))
return 0;
- f = fsh_try_alloc_fifo_hdr (fsh, fss);
- if (!f)
+ sf = fsh_try_alloc_fifo_hdr (fsh, fss);
+ if (!sf)
return 0;
c = fsh_try_alloc_chunk (fsh, fss, min_size);
if (!c)
{
- f->next = fss->free_fifos;
- fss->free_fifos = f;
+ sf->next = fss->free_fifos;
+ fss->free_fifos = sf;
return 0;
}
- f->start_chunk = c;
+ sf->start_chunk = c;
while (c->next)
c = c->next;
- f->end_chunk = c;
- f->size = data_bytes;
- f->fs_hdr = fsh;
+ sf->end_chunk = c;
+ sf->size = data_bytes;
+ sf->slice_index = slice_index;
- return f;
+ return sf;
}
svm_fifo_chunk_t *
fsh_slice_collect_chunks (fsh, fss, c);
}
+svm_fifo_t *
+fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index)
+{
+ fifo_slice_private_t *pfss = &fs->slices[slice_index];
+ svm_fifo_t *f;
+
+ f = clib_mem_bulk_alloc (pfss->fifos);
+ clib_memset (f, 0, sizeof (*f));
+ return f;
+}
+
+void
+fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f)
+{
+ u32 slice_index = f->shr->slice_index;
+ fifo_slice_private_t *pfss;
+
+ pfss = &fs->slices[slice_index];
+ clib_mem_bulk_free (pfss->fifos, f);
+}
+
+void
+fifo_segment_cleanup (fifo_segment_t *fs)
+{
+ int slice_index;
+
+ for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
+ clib_mem_bulk_destroy (fs->slices[slice_index].fifos);
+}
+
/**
* Allocate fifo in fifo segment
*/
{
fifo_segment_header_t *fsh = fs->h;
fifo_segment_slice_t *fss;
+ svm_fifo_shared_t *sf;
svm_fifo_t *f = 0;
ASSERT (slice_index < fs->n_slices);
if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size))
return 0;
- fss = fsh_slice_get (fsh, slice_index);
- f = fs_try_alloc_fifo (fsh, fss, data_bytes);
- if (!f)
+ sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes);
+ if (!sf)
goto done;
- f->slice_index = slice_index;
+ f = fs_fifo_alloc (fs, slice_index);
+ f->fs_hdr = fsh;
+ f->shr = sf;
svm_fifo_init (f, data_bytes);
+ fss = fsh_slice_get (fsh, slice_index);
+
/* If rx fifo type add to active fifos list. When cleaning up segment,
* we need a list of active sessions that should be disconnected. Since
* both rx and tx fifos keep pointers to the session, it's enough to track
return (f);
}
+svm_fifo_t *
+fifo_segment_alloc_fifo_w_shared (fifo_segment_t *fs, svm_fifo_shared_t *sf)
+{
+ svm_fifo_t *f = fs_fifo_alloc (fs, 0);
+ f->fs_hdr = fs->h;
+ f->shr = sf;
+
+ f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
+ f->refcnt = 1;
+ return f;
+}
+
/**
* Free fifo allocated in fifo segment
*/
{
fifo_segment_header_t *fsh = fs->h;
fifo_segment_slice_t *fss;
+ svm_fifo_shared_t *sf;
ASSERT (f->refcnt > 0);
if (--f->refcnt > 0)
return;
- fss = fsh_slice_get (fsh, f->slice_index);
+ /*
+ * Cleanup shared state
+ */
+
+ sf = f->shr;
+ fss = fsh_slice_get (fsh, sf->slice_index);
+
+ /* Free fifo chunks */
+ fsh_slice_collect_chunks (fsh, fss, sf->start_chunk);
+
+ sf->start_chunk = sf->end_chunk = 0;
+ sf->head_chunk = sf->tail_chunk = 0;
+
+ /* Add to free list */
+ sf->next = fss->free_fifos;
+ fss->free_fifos = sf;
+
+ fss->virtual_mem -= svm_fifo_size (f);
+
+ /*
+ * Cleanup private state
+ */
/* Remove from active list. Only rx fifos are tracked */
if (f->flags & SVM_FIFO_F_LL_TRACKED)
f->flags &= ~SVM_FIFO_F_LL_TRACKED;
}
- /* Free fifo chunks */
- fsh_slice_collect_chunks (fsh, fss, f->start_chunk);
-
- f->start_chunk = f->end_chunk = 0;
- f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
-
- /* not allocated on segment heap */
svm_fifo_free_chunk_lookup (f);
svm_fifo_free_ooo_data (f);
if (CLIB_DEBUG)
{
- f->master_session_index = ~0;
+ sf->master_session_index = ~0;
f->master_thread_index = ~0;
}
- fss->virtual_mem -= svm_fifo_size (f);
-
- /* Add to free list */
- f->next = fss->free_fifos;
+ f->ooo_enq = f->ooo_deq = 0;
f->prev = 0;
- fss->free_fifos = f;
+
+ fs_fifo_free (fs, f);
fsh_active_fifos_update (fsh, -1);
}
ASSERT (f->refcnt == 1);
- fss = fsh_slice_get (fs->h, f->slice_index);
+ fss = fsh_slice_get (fs->h, f->shr->slice_index);
fss->virtual_mem -= svm_fifo_size (f);
if (f->flags & SVM_FIFO_F_LL_TRACKED)
fss_fifo_del_active_list (fss, f);
- c = f->start_chunk;
+ c = f->shr->start_chunk;
while (c)
{
fl_index = fs_freelist_for_size (c->length);
svm_fifo_chunk_t *c;
u32 fl_index;
- f->slice_index = slice_index;
- fss = fsh_slice_get (fs->h, f->slice_index);
+ f->shr->slice_index = slice_index;
+ fss = fsh_slice_get (fs->h, f->shr->slice_index);
fss->virtual_mem += svm_fifo_size (f);
if (f->flags & SVM_FIFO_F_LL_TRACKED)
fss_fifo_add_active_list (fss, f);
- c = f->start_chunk;
+ c = f->shr->start_chunk;
while (c)
{
fl_index = fs_freelist_for_size (c->length);
static u32
fs_slice_num_free_fifos (fifo_segment_slice_t * fss)
{
- svm_fifo_t *f;
+ svm_fifo_shared_t *f;
u32 count = 0;
f = fss->free_fifos;
fifo_segment_header_t *h; /**< fifo segment data */
uword max_byte_index;
u8 n_slices; /**< number of fifo segment slices */
+ fifo_slice_private_t *slices; /**< private slice information */
} fifo_segment_t;
typedef struct
int fifo_segment_attach (fifo_segment_main_t * sm,
fifo_segment_create_args_t * a);
void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * fs);
+void fifo_segment_cleanup (fifo_segment_t *fs);
fifo_segment_t *fifo_segment_get_segment (fifo_segment_main_t * sm,
u32 fs_index);
u32 fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * fs);
u32 slice_index,
u32 data_bytes,
fifo_segment_ftype_t ftype);
+svm_fifo_t *fifo_segment_alloc_fifo_w_shared (fifo_segment_t *fs,
+ svm_fifo_shared_t *sf);
/**
* Free fifo allocated in fifo segment
u32 action;
} svm_fifo_trace_elem_t;
-typedef struct _svm_fifo
+typedef struct svm_fifo_shr_
{
- CLIB_CACHE_LINE_ALIGN_MARK (shared_first);
- fifo_segment_header_t *fs_hdr;/**< fifo segment header for fifo */
+ CLIB_CACHE_LINE_ALIGN_MARK (shared);
svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */
svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */
+ volatile u32 has_event; /**< non-zero if deq event exists */
u32 min_alloc; /**< min chunk alloc if space available */
u32 size; /**< size of the fifo in bytes */
- u8 flags; /**< fifo flags */
- u8 slice_index; /**< segment slice for fifo */
-
- CLIB_CACHE_LINE_ALIGN_MARK (shared_second);
- volatile u32 has_event; /**< non-zero if deq event exists */
u32 master_session_index; /**< session layer session index */
u32 client_session_index; /**< app session index */
- u8 master_thread_index; /**< session layer thread index */
- u8 client_thread_index; /**< app worker index */
- i8 refcnt; /**< reference count */
- u32 segment_manager; /**< session layer segment manager index */
- u32 segment_index; /**< segment index in segment manager */
- struct _svm_fifo *next; /**< next in freelist/active chain */
- struct _svm_fifo *prev; /**< prev in active chain */
-
- CLIB_CACHE_LINE_ALIGN_MARK (consumer);
- rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */
+ u8 slice_index; /**< segment slice for fifo */
+ struct svm_fifo_shr_ *next; /**< next in freelist/active chain */
+
+ CLIB_CACHE_LINE_ALIGN_MARK (consumer);
svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */
- svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
u32 head; /**< fifo head position/byte */
volatile u32 want_deq_ntf; /**< producer wants nudge */
volatile u32 has_deq_ntf;
- CLIB_CACHE_LINE_ALIGN_MARK (producer);
- rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */
+ CLIB_CACHE_LINE_ALIGN_MARK (producer);
u32 tail; /**< fifo tail position/byte */
- u32 ooos_list_head; /**< Head of out-of-order linked-list */
svm_fifo_chunk_t *tail_chunk; /**< tracks chunk where tail lands */
+ volatile u8 n_subscribers; /**< Number of subscribers for io events */
+ u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
+} svm_fifo_shared_t;
+
+typedef struct _svm_fifo
+{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
+ svm_fifo_shared_t *shr; /**< shared fifo in fifo segment memory */
+ fifo_segment_header_t *fs_hdr; /**< fifo segment header for fifo */
+ rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */
+ rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */
+ svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
+ u32 ooos_list_head; /**< Head of out-of-order linked-list */
u32 ooos_newest; /**< Last segment to have been updated */
- volatile u8 n_subscribers; /**< Number of subscribers for io events */
- u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
+
+ u8 flags; /**< fifo flags */
+ u8 master_thread_index; /**< session layer thread index */
+ u8 client_thread_index; /**< app worker index */
+ i8 refcnt; /**< reference count */
+ u32 segment_manager; /**< session layer segment manager index */
+ u32 segment_index; /**< segment index in segment manager */
+
+ struct _svm_fifo *next; /**< prev in active chain */
+ struct _svm_fifo *prev; /**< prev in active chain */
#if SVM_FIFO_TRACE
svm_fifo_trace_elem_t *trace;
#endif
-
} svm_fifo_t;
typedef struct fifo_segment_slice_
{
svm_fifo_chunk_t *free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
svm_fifo_t *fifos; /**< Linked list of active RX fifos */
- svm_fifo_t *free_fifos; /**< Freelists by fifo size */
+ svm_fifo_shared_t *free_fifos; /**< Freelists of fifo shared hdrs */
uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */
uword virtual_mem; /**< Slice sum of all fifo sizes */
u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */
u32 chunk_lock;
} fifo_segment_slice_t;
+typedef struct fifo_slice_private_
+{
+ clib_mem_bulk_handle_t fifos; /**< Bulk fifo allocator */
+ uword virtual_mem; /**< Slice sum of all fifo sizes */
+} fifo_slice_private_t;
+
struct fifo_segment_header_
{
uword n_cached_bytes; /**< Cached bytes */
}
}
- ASSERT (bytes <= f->size);
+ ASSERT (bytes <= f->shr->size);
return bytes;
}
svm_fifo_chunk_t *c, *prev;
u32 min_alloc;
- f->size = size;
+ f->shr->size = size;
f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
f->segment_index = SVM_FIFO_INVALID_INDEX;
f->refcnt = 1;
- f->head = f->tail = f->flags = 0;
- f->head_chunk = f->tail_chunk = f->start_chunk;
+ f->shr->head = f->shr->tail = f->flags = 0;
+ f->shr->head_chunk = f->shr->tail_chunk = f->shr->start_chunk;
f->ooo_deq = f->ooo_enq = 0;
min_alloc = size > 32 << 10 ? size >> 3 : 4096;
min_alloc = clib_min (min_alloc, 64 << 10);
- f->min_alloc = min_alloc;
+ f->shr->min_alloc = min_alloc;
/*
* Initialize chunks
*/
- f->start_chunk->start_byte = 0;
- prev = f->start_chunk;
+ f->shr->start_chunk->start_byte = 0;
+ prev = f->shr->start_chunk;
prev->enq_rb_index = prev->deq_rb_index = RBTREE_TNIL_INDEX;
c = prev->next;
c->length = data_size_in_bytes;
c->enq_rb_index = RBTREE_TNIL_INDEX;
c->deq_rb_index = RBTREE_TNIL_INDEX;
- f->start_chunk = f->end_chunk = c;
+ f->shr->start_chunk = f->shr->end_chunk = c;
return f;
}
{
svm_fifo_chunk_t *c;
- c = f->start_chunk;
+ c = f->shr->start_chunk;
while (c && !f_chunk_includes_pos (c, pos))
c = c->next;
u32 head, tail, end_chunk;
f_load_head_tail_cons (f, &head, &tail);
- ASSERT (!f->head_chunk || f_chunk_includes_pos (f->head_chunk, head));
+ ASSERT (!f->shr->head_chunk ||
+ f_chunk_includes_pos (f->shr->head_chunk, head));
- if (!f->head_chunk)
+ if (!f->shr->head_chunk)
{
- f->head_chunk = svm_fifo_find_chunk (f, head);
- if (PREDICT_FALSE (!f->head_chunk))
+ f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+ if (PREDICT_FALSE (!f->shr->head_chunk))
return 0;
}
- end_chunk = f_chunk_end (f->head_chunk);
+ end_chunk = f_chunk_end (f->shr->head_chunk);
return f_pos_lt (end_chunk, tail) ? end_chunk - head : tail - head;
}
u32 head, tail;
f_load_head_tail_prod (f, &head, &tail);
- ASSERT (!f->tail_chunk || f_chunk_includes_pos (f->tail_chunk, tail));
+ ASSERT (!f->shr->tail_chunk ||
+ f_chunk_includes_pos (f->shr->tail_chunk, tail));
- return f->tail_chunk ? f_chunk_end (f->tail_chunk) - tail : 0;
+ return f->shr->tail_chunk ? f_chunk_end (f->shr->tail_chunk) - tail : 0;
}
static rb_node_t *
/* Use linear search if rbtree is not initialized */
if (PREDICT_FALSE (!rb_tree_is_init (rt)))
{
- f->ooo_enq = svm_fifo_find_next_chunk (f, f->tail_chunk, start_pos);
+ f->ooo_enq = svm_fifo_find_next_chunk (f, f->shr->tail_chunk, start_pos);
return;
}
if (rt->root == RBTREE_TNIL_INDEX)
{
- c = f->tail_chunk;
+ c = f->shr->tail_chunk;
ASSERT (c->enq_rb_index == RBTREE_TNIL_INDEX);
c->enq_rb_index = rb_tree_add_custom (rt, c->start_byte,
pointer_to_uword (c), f_pos_lt);
if (rt->root == RBTREE_TNIL_INDEX)
{
- c = f->start_chunk;
+ c = f->shr->start_chunk;
ASSERT (c->deq_rb_index == RBTREE_TNIL_INDEX);
c->deq_rb_index = rb_tree_add_custom (rt, c->start_byte,
pointer_to_uword (c), f_pos_lt);
u32 head, tail, head_idx;
svm_fifo_chunk_t *c;
- ASSERT (len <= f->size);
+ ASSERT (len <= f->shr->size);
f_load_head_tail_cons (f, &head, &tail);
- if (!f->head_chunk)
- f->head_chunk = svm_fifo_find_chunk (f, head);
+ if (!f->shr->head_chunk)
+ f->shr->head_chunk = svm_fifo_find_chunk (f, head);
- c = f->head_chunk;
+ c = f->shr->head_chunk;
head_idx = head - c->start_byte;
n_chunk = c->length - head_idx;
if (len <= n_chunk)
svm_fifo_chunk_t *c, *cur, *prev;
u32 alloc_size, free_alloced;
- free_alloced = f_chunk_end (f->end_chunk) - tail;
+ free_alloced = f_chunk_end (f->shr->end_chunk) - tail;
- alloc_size = clib_min (f->min_alloc, f->size - (tail - head));
+ alloc_size = clib_min (f->shr->min_alloc, f->shr->size - (tail - head));
alloc_size = clib_max (alloc_size, len - free_alloced);
- c = fsh_alloc_chunk (f->fs_hdr, f->slice_index, alloc_size);
+ c = fsh_alloc_chunk (f->fs_hdr, f->shr->slice_index, alloc_size);
if (PREDICT_FALSE (!c))
return -1;
cur = c;
- prev = f->end_chunk;
+ prev = f->shr->end_chunk;
while (cur)
{
}
prev->next = 0;
- f->end_chunk->next = c;
- f->end_chunk = prev;
+ f->shr->end_chunk->next = c;
+ f->shr->end_chunk = prev;
- if (!f->tail_chunk)
- f->tail_chunk = c;
+ if (!f->shr->tail_chunk)
+ f->shr->tail_chunk = c;
return 0;
}
/* number of bytes we're going to copy */
len = clib_min (free_count, len);
- if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk)))
+ if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
{
if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
{
- len = f_chunk_end (f->end_chunk) - tail;
+ len = f_chunk_end (f->shr->end_chunk) - tail;
if (!len)
return SVM_FIFO_EGROW;
}
}
- old_tail_c = f->tail_chunk;
+ old_tail_c = f->shr->tail_chunk;
- svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, src, len, &f->tail_chunk);
+ svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, src, len,
+ &f->shr->tail_chunk);
tail = tail + len;
svm_fifo_trace_add (f, head, len, 2);
{
len += ooo_segment_try_collect (f, len, &tail);
/* Tail chunk might've changed even if nothing was collected */
- f->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
+ f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
f->ooo_enq = 0;
}
/* store-rel: producer owned index (paired with load-acq in consumer) */
- clib_atomic_store_rel_n (&f->tail, tail);
+ clib_atomic_store_rel_n (&f->shr->tail, tail);
return len;
}
enq_pos = tail + offset;
- if (f_pos_gt (enq_pos + len, f_chunk_end (f->end_chunk)))
+ if (f_pos_gt (enq_pos + len, f_chunk_end (f->shr->end_chunk)))
{
if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, offset + len)))
return SVM_FIFO_EGROW;
ASSERT (len <= svm_fifo_max_enqueue_prod (f));
/* load-relaxed: producer owned index */
- tail = f->tail;
+ tail = f->shr->tail;
tail = tail + len;
if (rb_tree_is_init (&f->ooo_enq_lookup))
{
- f->tail_chunk = f_lookup_clear_enq_chunks (f, f->tail_chunk, tail);
+ f->shr->tail_chunk =
+ f_lookup_clear_enq_chunks (f, f->shr->tail_chunk, tail);
f->ooo_enq = 0;
}
else
{
- f->tail_chunk = svm_fifo_find_next_chunk (f, f->tail_chunk, tail);
+ f->shr->tail_chunk =
+ svm_fifo_find_next_chunk (f, f->shr->tail_chunk, tail);
}
/* store-rel: producer owned index (paired with load-acq in consumer) */
- clib_atomic_store_rel_n (&f->tail, tail);
+ clib_atomic_store_rel_n (&f->shr->tail, tail);
}
int
for (i = 0; i < n_segs; i++)
len += segs[i].len;
- old_tail_c = f->tail_chunk;
+ old_tail_c = f->shr->tail_chunk;
if (!allow_partial)
{
if (PREDICT_FALSE (free_count < len))
return SVM_FIFO_EFULL;
- if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk)))
+ if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
{
if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
return SVM_FIFO_EGROW;
for (i = 0; i < n_segs; i++)
{
- svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data,
- segs[i].len, &f->tail_chunk);
+ svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data,
+ segs[i].len, &f->shr->tail_chunk);
tail += segs[i].len;
}
}
{
len = clib_min (free_count, len);
- if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk)))
+ if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
{
if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
{
- len = f_chunk_end (f->end_chunk) - tail;
+ len = f_chunk_end (f->shr->end_chunk) - tail;
if (!len)
return SVM_FIFO_EGROW;
}
while (len)
{
u32 to_copy = clib_min (segs[i].len, len);
- svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data,
- to_copy, &f->tail_chunk);
+ svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data,
+ to_copy, &f->shr->tail_chunk);
len -= to_copy;
tail += to_copy;
i++;
{
len += ooo_segment_try_collect (f, len, &tail);
/* Tail chunk might've changed even if nothing was collected */
- f->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
+ f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
f->ooo_enq = 0;
}
/* store-rel: producer owned index (paired with load-acq in consumer) */
- clib_atomic_store_rel_n (&f->tail, tail);
+ clib_atomic_store_rel_n (&f->shr->tail, tail);
return len;
}
rb_tree_t *rt;
rb_node_t *n;
- ASSERT (!f_chunk_includes_pos (f->start_chunk, end_pos));
+ ASSERT (!f_chunk_includes_pos (f->shr->start_chunk, end_pos));
if (maybe_ooo)
rt = &f->ooo_deq_lookup;
- c = f->start_chunk;
+ c = f->shr->start_chunk;
do
{
return 0;
prev->next = 0;
- start = f->start_chunk;
- f->start_chunk = c;
+ start = f->shr->start_chunk;
+ f->shr->start_chunk = c;
return start;
}
len = clib_min (cursize, len);
- if (!f->head_chunk)
- f->head_chunk = svm_fifo_find_chunk (f, head);
+ if (!f->shr->head_chunk)
+ f->shr->head_chunk = svm_fifo_find_chunk (f, head);
- svm_fifo_copy_from_chunk (f, f->head_chunk, head, dst, len, &f->head_chunk);
+ svm_fifo_copy_from_chunk (f, f->shr->head_chunk, head, dst, len,
+ &f->shr->head_chunk);
head = head + len;
/* In order dequeues are not supported in combination with ooo peeking.
* Use svm_fifo_dequeue_drop instead. */
ASSERT (rb_tree_n_nodes (&f->ooo_deq_lookup) <= 1);
- if (f_pos_geq (head, f_chunk_end (f->start_chunk)))
- fsh_collect_chunks (f->fs_hdr, f->slice_index,
+ if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk)))
+ fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
f_unlink_chunks (f, head, 0));
/* store-rel: consumer owned index (paired with load-acq in producer) */
- clib_atomic_store_rel_n (&f->head, head);
+ clib_atomic_store_rel_n (&f->shr->head, head);
return len;
}
/* move head */
head = head + total_drop_bytes;
- if (f_pos_geq (head, f_chunk_end (f->start_chunk)))
+ if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk)))
{
- fsh_collect_chunks (f->fs_hdr, f->slice_index,
+ fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
f_unlink_chunks (f, head, 1));
- f->head_chunk =
- f_chunk_includes_pos (f->start_chunk, head) ? f->start_chunk : 0;
+ f->shr->head_chunk = f_chunk_includes_pos (f->shr->start_chunk, head) ?
+ f->shr->start_chunk :
+ 0;
}
/* store-rel: consumer owned index (paired with load-acq in producer) */
- clib_atomic_store_rel_n (&f->head, head);
+ clib_atomic_store_rel_n (&f->shr->head, head);
return total_drop_bytes;
}
f_load_head_tail_all_acq (f, &head, &tail);
- if (!f->head_chunk || !f_chunk_includes_pos (f->head_chunk, head))
- f->head_chunk = svm_fifo_find_chunk (f, head);
+ if (!f->shr->head_chunk || !f_chunk_includes_pos (f->shr->head_chunk, head))
+ f->shr->head_chunk = svm_fifo_find_chunk (f, head);
- f->head_chunk = f_lookup_clear_deq_chunks (f, f->head_chunk, tail);
+ f->shr->head_chunk = f_lookup_clear_deq_chunks (f, f->shr->head_chunk, tail);
- if (f_pos_geq (tail, f_chunk_end (f->start_chunk)))
- fsh_collect_chunks (f->fs_hdr, f->slice_index,
+ if (f_pos_geq (tail, f_chunk_end (f->shr->start_chunk)))
+ fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
f_unlink_chunks (f, tail, 0));
/* store-rel: consumer owned index (paired with load-acq in producer) */
- clib_atomic_store_rel_n (&f->head, tail);
+ clib_atomic_store_rel_n (&f->shr->head, tail);
}
int
f_load_head_tail_prod (f, &head, &tail);
- if (f_chunk_end (f->end_chunk) - head >= f->size)
+ if (f_chunk_end (f->shr->end_chunk) - head >= f->shr->size)
return 0;
- if (f_try_chunk_alloc (f, head, tail, f->size - (tail - head)))
+ if (f_try_chunk_alloc (f, head, tail, f->shr->size - (tail - head)))
return SVM_FIFO_EGROW;
return 0;
if (f_free_count (f, head, tail) < len)
return SVM_FIFO_EFULL;
- n_avail = f_chunk_end (f->end_chunk) - tail;
+ n_avail = f_chunk_end (f->shr->end_chunk) - tail;
if (n_avail < len && f_try_chunk_alloc (f, head, tail, len))
return SVM_FIFO_EGROW;
- c = f->tail_chunk;
+ c = f->shr->tail_chunk;
head_pos = (tail - c->start_byte);
fs[0].data = c->data + head_pos;
fs[0].len = clib_min (c->length - head_pos, len);
to_read = clib_min (cursize - offset, max_bytes);
start = head + offset;
- if (!f->head_chunk)
- f->head_chunk = svm_fifo_find_chunk (f, head);
+ if (!f->shr->head_chunk)
+ f->shr->head_chunk = svm_fifo_find_chunk (f, head);
- c = f->head_chunk;
+ c = f->shr->head_chunk;
while (!f_chunk_includes_pos (c, start))
c = c->next;
/* Support only single chunk clones for now */
ASSERT (svm_fifo_n_chunks (sf) == 1);
- clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size);
+ clib_memcpy_fast (df->shr->head_chunk->data, sf->shr->head_chunk->data,
+ sf->shr->size);
f_load_head_tail_all_acq (sf, &head, &tail);
- clib_atomic_store_rel_n (&df->head, head);
- clib_atomic_store_rel_n (&df->tail, tail);
+ clib_atomic_store_rel_n (&df->shr->head, head);
+ clib_atomic_store_rel_n (&df->shr->tail, tail);
}
u32
{
svm_fifo_chunk_t *c;
- clib_atomic_store_rel_n (&f->head, head);
- clib_atomic_store_rel_n (&f->tail, tail);
+ clib_atomic_store_rel_n (&f->shr->head, head);
+ clib_atomic_store_rel_n (&f->shr->tail, tail);
c = svm_fifo_find_chunk (f, head);
ASSERT (c != 0);
- f->head_chunk = f->ooo_deq = c;
+ f->shr->head_chunk = f->ooo_deq = c;
c = svm_fifo_find_chunk (f, tail);
ASSERT (c != 0);
- f->tail_chunk = f->ooo_enq = c;
+ f->shr->tail_chunk = f->ooo_enq = c;
}
void
svm_fifo_add_subscriber (svm_fifo_t * f, u8 subscriber)
{
- if (f->n_subscribers >= SVM_FIFO_MAX_EVT_SUBSCRIBERS)
+ if (f->shr->n_subscribers >= SVM_FIFO_MAX_EVT_SUBSCRIBERS)
return;
- f->subscribers[f->n_subscribers++] = subscriber;
+ f->shr->subscribers[f->shr->n_subscribers++] = subscriber;
}
void
{
int i;
- for (i = 0; i < f->n_subscribers; i++)
+ for (i = 0; i < f->shr->n_subscribers; i++)
{
- if (f->subscribers[i] != subscriber)
+ if (f->shr->subscribers[i] != subscriber)
continue;
- f->subscribers[i] = f->subscribers[f->n_subscribers - 1];
- f->n_subscribers--;
+ f->shr->subscribers[i] = f->shr->subscribers[f->shr->n_subscribers - 1];
+ f->shr->n_subscribers--;
break;
}
}
{
svm_fifo_chunk_t *tmp;
- if (f->head_chunk && !f_chunk_includes_pos (f->head_chunk, f->head))
+ if (f->shr->head_chunk &&
+ !f_chunk_includes_pos (f->shr->head_chunk, f->shr->head))
return 0;
- if (f->tail_chunk && !f_chunk_includes_pos (f->tail_chunk, f->tail))
+ if (f->shr->tail_chunk &&
+ !f_chunk_includes_pos (f->shr->tail_chunk, f->shr->tail))
return 0;
if (f->ooo_deq)
{
if (rb_tree_is_init (&f->ooo_deq_lookup))
{
- if (f_pos_lt (f->ooo_deq->start_byte, f->start_chunk->start_byte)
- || f_pos_gt (f->ooo_deq->start_byte,
- f_chunk_end (f->end_chunk)))
+ if (f_pos_lt (f->ooo_deq->start_byte,
+ f->shr->start_chunk->start_byte) ||
+ f_pos_gt (f->ooo_deq->start_byte,
+ f_chunk_end (f->shr->end_chunk)))
return 0;
tmp = f_find_chunk_rbtree (&f->ooo_deq_lookup,
{
if (rb_tree_is_init (&f->ooo_enq_lookup))
{
- if (f_pos_lt (f->ooo_enq->start_byte, f->start_chunk->start_byte)
- || f_pos_gt (f->ooo_enq->start_byte,
- f_chunk_end (f->end_chunk)))
+ if (f_pos_lt (f->ooo_enq->start_byte,
+ f->shr->start_chunk->start_byte) ||
+ f_pos_gt (f->ooo_enq->start_byte,
+ f_chunk_end (f->shr->end_chunk)))
return 0;
tmp = f_find_chunk_rbtree (&f->ooo_enq_lookup,
}
else
{
- tmp = svm_fifo_find_next_chunk (f, f->tail_chunk,
+ tmp = svm_fifo_find_next_chunk (f, f->shr->tail_chunk,
f->ooo_enq->start_byte);
}
if (tmp != f->ooo_enq)
return 0;
}
- if (f->start_chunk->next)
+ if (f->shr->start_chunk->next)
{
svm_fifo_chunk_t *c, *prev = 0, *tmp;
u32 chunks_bytes = 0;
- c = f->start_chunk;
+ c = f->shr->start_chunk;
do
{
tmp = svm_fifo_find_chunk (f, c->start_byte);
}
while (c);
- if (chunks_bytes < f->tail - f->head)
+ if (chunks_bytes < f->shr->tail - f->shr->head)
return 0;
}
svm_fifo_chunk_t *c;
int n_chunks = 0;
- c = f->start_chunk;
+ c = f->shr->start_chunk;
while (c)
{
n_chunks++;
trace_len = 0;
#endif
- placeholder_fifo = svm_fifo_alloc (f->size);
- svm_fifo_init (f, f->size);
- clib_memset (f->head_chunk->data, 0xFF, f->size);
- vec_validate (data, f->size);
+ placeholder_fifo = svm_fifo_alloc (f->shr->size);
+ svm_fifo_init (f, f->shr->size);
+ clib_memset (f->shr->head_chunk->data, 0xFF, f->shr->size);
+ vec_validate (data, f->shr->size);
for (i = 0; i < vec_len (data); i++)
data[i] = i;
indent = format_get_indent (s);
s = format (s, "cursize %u nitems %u has_event %d min_alloc %u\n",
- svm_fifo_max_dequeue (f), f->size, f->has_event, f->min_alloc);
+ svm_fifo_max_dequeue (f), f->shr->size, f->shr->has_event,
+ f->shr->min_alloc);
s = format (s, "%Uhead %u tail %u segment manager %u\n", format_white_space,
- indent, f->head, f->tail, f->segment_manager);
+ indent, f->shr->head, f->shr->tail, f->segment_manager);
if (verbose > 1)
s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n",
- format_white_space, indent, f->master_session_index,
- f->master_thread_index, f->client_session_index,
+ format_white_space, indent, f->shr->master_session_index,
+ f->master_thread_index, f->shr->client_session_index,
f->client_thread_index);
if (verbose)
f_load_head_tail_cons (svm_fifo_t * f, u32 * head, u32 * tail)
{
/* load-relaxed: consumer owned index */
- *head = f->head;
+ *head = f->shr->head;
/* load-acq: consumer foreign index (paired with store-rel in producer) */
- *tail = clib_atomic_load_acq_n (&f->tail);
+ *tail = clib_atomic_load_acq_n (&f->shr->tail);
}
/** Load head and tail optimized for producer
f_load_head_tail_prod (svm_fifo_t * f, u32 * head, u32 * tail)
{
/* load relaxed: producer owned index */
- *tail = f->tail;
+ *tail = f->shr->tail;
/* load-acq: producer foreign index (paired with store-rel in consumer) */
- *head = clib_atomic_load_acq_n (&f->head);
+ *head = clib_atomic_load_acq_n (&f->shr->head);
}
/**
f_load_head_tail_all_acq (svm_fifo_t * f, u32 * head, u32 * tail)
{
/* load-acq : consumer foreign index (paired with store-rel) */
- *tail = clib_atomic_load_acq_n (&f->tail);
+ *tail = clib_atomic_load_acq_n (&f->shr->tail);
/* load-acq : producer foriegn index (paired with store-rel) */
- *head = clib_atomic_load_acq_n (&f->head);
+ *head = clib_atomic_load_acq_n (&f->shr->head);
}
/**
static inline u32
f_free_count (svm_fifo_t * f, u32 head, u32 tail)
{
- return (f->size - f_cursize (f, head, tail));
+ return (f->shr->size - f_cursize (f, head, tail));
}
always_inline u32
static inline int
svm_fifo_is_full_prod (svm_fifo_t * f)
{
- return (svm_fifo_max_dequeue_prod (f) == f->size);
+ return (svm_fifo_max_dequeue_prod (f) == f->shr->size);
}
/* Check if fifo is full.
static inline int
svm_fifo_is_full (svm_fifo_t * f)
{
- return (svm_fifo_max_dequeue (f) == f->size);
+ return (svm_fifo_max_dequeue (f) == f->shr->size);
}
/**
static inline svm_fifo_chunk_t *
svm_fifo_head_chunk (svm_fifo_t * f)
{
- return f->head_chunk;
+ return f->shr->head_chunk;
}
/**
static inline u8 *
svm_fifo_head (svm_fifo_t * f)
{
- if (!f->head_chunk)
+ if (!f->shr->head_chunk)
return 0;
/* load-relaxed: consumer owned index */
- return (f->head_chunk->data + (f->head - f->head_chunk->start_byte));
+ return (f->shr->head_chunk->data +
+ (f->shr->head - f->shr->head_chunk->start_byte));
}
/**
static inline svm_fifo_chunk_t *
svm_fifo_tail_chunk (svm_fifo_t * f)
{
- return f->tail_chunk;
+ return f->shr->tail_chunk;
}
/**
svm_fifo_tail (svm_fifo_t * f)
{
/* load-relaxed: producer owned index */
- return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte));
+ return (f->shr->tail_chunk->data +
+ (f->shr->tail - f->shr->tail_chunk->start_byte));
}
/**
static inline u8
svm_fifo_n_subscribers (svm_fifo_t * f)
{
- return f->n_subscribers;
+ return f->shr->n_subscribers;
}
/**
{
u32 tail;
/* load-relaxed: producer owned index */
- tail = f->tail;
+ tail = f->shr->tail;
return (s->start - tail);
}
static inline u32
svm_fifo_size (svm_fifo_t * f)
{
- return f->size;
+ return f->shr->size;
}
static inline void
{
if (size > (1 << f->fs_hdr->max_log2_fifo_size))
return;
- fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size);
- f->size = size;
+ fsh_virtual_mem_update (f->fs_hdr, f->shr->slice_index,
+ (int) f->shr->size - size);
+ f->shr->size = size;
}
/**
static inline int
svm_fifo_has_event (svm_fifo_t * f)
{
- return f->has_event;
+ return f->shr->has_event;
}
/**
always_inline u8
svm_fifo_set_event (svm_fifo_t * f)
{
- return !clib_atomic_swap_rel_n (&f->has_event, 1);
+ return !clib_atomic_swap_rel_n (&f->shr->has_event, 1);
}
/**
always_inline void
svm_fifo_unset_event (svm_fifo_t * f)
{
- clib_atomic_swap_acq_n (&f->has_event, 0);
+ clib_atomic_swap_acq_n (&f->shr->has_event, 0);
}
/**
static inline void
svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
{
- f->want_deq_ntf |= ntf_type;
+ f->shr->want_deq_ntf |= ntf_type;
}
/**
static inline void
svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
{
- f->want_deq_ntf &= ~ntf_type;
+ f->shr->want_deq_ntf &= ~ntf_type;
}
/**
svm_fifo_clear_deq_ntf (svm_fifo_t * f)
{
/* Set the flag if want_notif_if_full was the only ntf requested */
- f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL;
+ f->shr->has_deq_ntf =
+ f->shr->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL;
svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF);
}
static inline void
svm_fifo_reset_has_deq_ntf (svm_fifo_t * f)
{
- f->has_deq_ntf = 0;
+ f->shr->has_deq_ntf = 0;
}
/**
static inline u8
svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq)
{
- u8 want_ntf = f->want_deq_ntf;
+ u8 want_ntf = f->shr->want_deq_ntf;
if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF))
return 0;
if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL)
{
u32 max_deq = svm_fifo_max_dequeue_cons (f);
- u32 size = f->size;
- if (!f->has_deq_ntf && max_deq < size && max_deq + n_last_deq >= size)
+ u32 size = f->shr->size;
+ if (!f->shr->has_deq_ntf && max_deq < size &&
+ max_deq + n_last_deq >= size)
return 1;
}
if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY)
{
- if (!f->has_deq_ntf && svm_fifo_is_empty (f))
+ if (!f->shr->has_deq_ntf && svm_fifo_is_empty (f))
return 1;
}
return 0;
VDBG (0, "detached segment %u handle %u", segment_index, segment_handle);
}
+int
+vcl_segment_attach_session (uword segment_handle, uword rxf_offset,
+ uword txf_offset, u8 is_ct, vcl_session_t *s)
+{
+ svm_fifo_shared_t *rxsf, *txsf;
+ svm_fifo_t *rxf, *txf;
+ fifo_segment_t *fs;
+ u32 fs_index;
+
+ fs_index = vcl_segment_table_lookup (segment_handle);
+ if (fs_index == VCL_INVALID_SEGMENT_INDEX)
+ {
+ VDBG (0, "ERROR: segment for session %u is not mounted!",
+ s->session_index);
+ return -1;
+ }
+
+ rxsf = uword_to_pointer (rxf_offset, svm_fifo_shared_t *);
+ txsf = uword_to_pointer (txf_offset, svm_fifo_shared_t *);
+
+ clib_rwlock_reader_lock (&vcm->segment_table_lock);
+
+ fs = fifo_segment_get_segment (&vcm->segment_main, fs_index);
+ rxf = fifo_segment_alloc_fifo_w_shared (fs, rxsf);
+ txf = fifo_segment_alloc_fifo_w_shared (fs, txsf);
+
+ clib_rwlock_reader_unlock (&vcm->segment_table_lock);
+
+ if (!is_ct)
+ {
+ rxsf->client_session_index = s->session_index;
+ txsf->client_session_index = s->session_index;
+ rxf->client_thread_index = vcl_get_worker_index ();
+ txf->client_thread_index = vcl_get_worker_index ();
+ s->rx_fifo = rxf;
+ s->tx_fifo = txf;
+ }
+ else
+ {
+ s->ct_rx_fifo = rxf;
+ s->ct_tx_fifo = txf;
+ }
+
+ return 0;
+}
/*
* fd.io coding-style-patch-verification: ON
void vcl_segment_detach (u64 segment_handle);
void vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s);
+int vcl_segment_attach_session (uword segment_handle, uword rxf_offset,
+ uword txf_offset, u8 is_ct, vcl_session_t *s);
+
/*
* VCL Binary API
*/
__thread uword __vcl_worker_index = ~0;
-static int
-vcl_segment_is_not_mounted (vcl_worker_t * wrk, u64 segment_handle)
-{
- u32 segment_index;
-
- if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
- return 0;
-
- segment_index = vcl_segment_table_lookup (segment_handle);
- if (segment_index != VCL_INVALID_SEGMENT_INDEX)
- return 0;
-
- return 1;
-}
-
static inline int
vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg)
{
u32 ls_index)
{
vcl_session_t *session, *listen_session;
- svm_fifo_t *rx_fifo, *tx_fifo;
svm_msg_q_t *evt_q;
session = vcl_session_alloc (wrk);
goto error;
}
- if (vcl_segment_is_not_mounted (wrk, mp->segment_handle))
+ session->vpp_evt_q =
+ uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *);
+
+ if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo,
+ mp->server_tx_fifo, 0, session))
{
- VDBG (0, "ERROR: segment for session %u is not mounted!",
- session->session_index);
+ VDBG (0, "failed to attach fifos for %u", session->session_index);
goto error;
}
- rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
- tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
- session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
- svm_msg_q_t *);
- rx_fifo->client_session_index = session->session_index;
- tx_fifo->client_session_index = session->session_index;
- rx_fifo->client_thread_index = vcl_get_worker_index ();
- tx_fifo->client_thread_index = vcl_get_worker_index ();
-
session->vpp_handle = mp->handle;
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
-
session->session_state = VCL_STATE_READY;
session->transport.rmt_port = mp->rmt.port;
session->transport.is_ip4 = mp->rmt.is_ip4;
vcl_session_connected_handler (vcl_worker_t * wrk,
session_connected_msg_t * mp)
{
- svm_fifo_t *rx_fifo, *tx_fifo;
vcl_session_t *session = 0;
u32 session_index;
session->vpp_handle = mp->handle;
session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address,
svm_msg_q_t *);
- rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *);
- tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *);
- if (vcl_segment_is_not_mounted (wrk, mp->segment_handle))
+
+ if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo,
+ mp->server_tx_fifo, 0, session))
{
- VDBG (0, "segment for session %u is not mounted!",
- session->session_index);
+ VDBG (0, "failed to attach fifos for %u", session->session_index);
session->session_state = VCL_STATE_DETACHED;
vcl_send_session_disconnect (wrk, session);
return session_index;
}
- rx_fifo->client_session_index = session_index;
- tx_fifo->client_session_index = session_index;
- rx_fifo->client_thread_index = vcl_get_worker_index ();
- tx_fifo->client_thread_index = vcl_get_worker_index ();
-
if (mp->ct_rx_fifo)
{
- session->ct_rx_fifo = uword_to_pointer (mp->ct_rx_fifo, svm_fifo_t *);
- session->ct_tx_fifo = uword_to_pointer (mp->ct_tx_fifo, svm_fifo_t *);
- if (vcl_segment_is_not_mounted (wrk, mp->ct_segment_handle))
+ if (vcl_segment_attach_session (mp->ct_segment_handle, mp->ct_rx_fifo,
+ mp->ct_tx_fifo, 1, session))
{
- VDBG (0, "ct segment for session %u is not mounted!",
- session->session_index);
+ VDBG (0, "failed to attach ct fifos for %u", session->session_index);
session->session_state = VCL_STATE_DETACHED;
vcl_send_session_disconnect (wrk, session);
return session_index;
}
}
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
session->transport.is_ip4 = mp->lcl.is_ip4;
clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
sizeof (session->transport.lcl_ip));
if (vcl_session_is_cl (session))
{
- svm_fifo_t *rx_fifo, *tx_fifo;
- session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
- rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *);
- rx_fifo->client_session_index = sid;
- tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *);
- tx_fifo->client_session_index = sid;
- session->rx_fifo = rx_fifo;
- session->tx_fifo = tx_fifo;
+ if (vcl_segment_attach_session (mp->segment_handle, mp->rx_fifo,
+ mp->tx_fifo, 0, session))
+ {
+ VDBG (0, "failed to attach fifos for %u", session->session_index);
+ session->session_state = VCL_STATE_DETACHED;
+ return VCL_INVALID_SESSION_INDEX;
+ }
}
VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle);
{
session_migrated_msg_t *mp = (session_migrated_msg_t *) data;
vcl_session_t *s;
+ u32 fs_index;
s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
if (!s)
return;
}
+ fs_index = vcl_segment_table_lookup (mp->segment_handle);
+ if (fs_index == VCL_INVALID_SEGMENT_INDEX)
+ {
+ VDBG (0, "segment for session %u is not mounted!", s->session_index);
+ s->session_state = VCL_STATE_DETACHED;
+ return;
+ }
+
s->vpp_handle = mp->new_handle;
s->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *);
/* Generate new tx event if we have outstanding data */
if (svm_fifo_has_event (s->tx_fifo))
- app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index,
+ app_send_io_evt_to_vpp (s->vpp_evt_q,
+ s->tx_fifo->shr->master_session_index,
SESSION_IO_EVT_TX, SVM_Q_WAIT);
VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle,
VDBG (0, "unknown handle 0x%llx", msg->handle);
return;
}
- if (vcl_segment_is_not_mounted (wrk, msg->segment_handle))
- {
- clib_warning ("segment for session %u is not mounted!",
- s->session_index);
- return;
- }
if (s->rx_fifo)
{
- s->rx_fifo = uword_to_pointer (msg->rx_fifo, svm_fifo_t *);
- s->tx_fifo = uword_to_pointer (msg->tx_fifo, svm_fifo_t *);
- s->rx_fifo->client_session_index = s->session_index;
- s->tx_fifo->client_session_index = s->session_index;
- s->rx_fifo->client_thread_index = wrk->wrk_index;
- s->tx_fifo->client_thread_index = wrk->wrk_index;
+ if (vcl_segment_attach_session (msg->segment_handle, msg->rx_fifo,
+ msg->tx_fifo, 0, s))
+ {
+ VDBG (0, "failed to attach fifos for %u", s->session_index);
+ return;
+ }
}
s->session_state = VCL_STATE_UPDATED;
if (PREDICT_FALSE (svm_fifo_needs_deq_ntf (rx_fifo, n_read)))
{
svm_fifo_clear_deq_ntf (rx_fifo);
- app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo->master_session_index,
+ app_send_io_evt_to_vpp (s->vpp_evt_q,
+ s->rx_fifo->shr->master_session_index,
SESSION_IO_EVT_RX, SVM_Q_WAIT);
}
0 /* do_evt */ , SVM_Q_WAIT);
if (svm_fifo_set_event (s->tx_fifo))
- app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index,
- et, SVM_Q_WAIT);
+ app_send_io_evt_to_vpp (
+ s->vpp_evt_q, s->tx_fifo->shr->master_session_index, et, SVM_Q_WAIT);
/* The underlying fifo segment can run out of memory */
if (PREDICT_FALSE (n_write < 0))
uword rx_fifo;
uword tx_fifo;
uword vpp_evt_q;
- u32 segment_size;
- u8 segment_name_length;
- u8 segment_name[128];
+ u64 segment_handle;
} __clib_packed session_bound_msg_t;
typedef struct session_unlisten_msg_
uword vpp_evt_q;
session_handle_t handle;
session_handle_t new_handle;
+ u64 segment_handle;
u32 vpp_thread_index;
} __clib_packed session_migrated_msg_t;
if (do_evt)
{
if (svm_fifo_set_event (f))
- app_send_io_evt_to_vpp (vpp_evt_q, f->master_session_index, evt_type,
- noblock);
+ app_send_io_evt_to_vpp (vpp_evt_q, f->shr->master_session_index,
+ evt_type, noblock);
}
return len;
}
if (do_evt)
{
if (rv > 0 && svm_fifo_set_event (f))
- app_send_io_evt_to_vpp (vpp_evt_q, f->master_session_index, evt_type,
- noblock);
+ app_send_io_evt_to_vpp (vpp_evt_q, f->shr->master_session_index,
+ evt_type, noblock);
}
return rv;
}
}
sm_index = segment_manager_index (sm);
- ls->rx_fifo->master_session_index = ls->session_index;
- ls->tx_fifo->master_session_index = ls->session_index;
+ ls->rx_fifo->shr->master_session_index = ls->session_index;
+ ls->tx_fifo->shr->master_session_index = ls->session_index;
ls->rx_fifo->master_thread_index = ls->thread_index;
ls->tx_fifo->master_thread_index = ls->thread_index;
ls->rx_fifo->segment_manager = sm_index;
&rx_fifo, &tx_fifo)))
return rv;
- rx_fifo->master_session_index = s->session_index;
+ rx_fifo->shr->master_session_index = s->session_index;
rx_fifo->master_thread_index = s->thread_index;
- tx_fifo->master_session_index = s->session_index;
+ tx_fifo->shr->master_session_index = s->session_index;
tx_fifo->master_thread_index = s->thread_index;
s->rx_fifo = rx_fifo;
msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
- evt->session_index = s->rx_fifo->client_session_index;
+ evt->session_index = s->rx_fifo->shr->client_session_index;
evt->event_type = SESSION_IO_EVT_RX;
(void) svm_fifo_set_event (s->rx_fifo);
msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
evt->event_type = SESSION_IO_EVT_TX;
- evt->session_index = s->tx_fifo->client_session_index;
+ evt->session_index = s->tx_fifo->shr->client_session_index;
svm_msg_q_add_and_unlock (mq, &msg);
return 0;
}
}
+ fifo_segment_cleanup (fs);
ssvm_delete (&fs->ssvm);
if (CLIB_DEBUG)
*/
while (f)
{
- session = session_get_if_valid (f->master_session_index,
- f->master_thread_index);
- if (session)
- vec_add1 (handles, session_handle (session));
- f = f->next;
- }
+ session = session_get_if_valid (f->shr->master_session_index,
+ f->master_thread_index);
+ if (session)
+ vec_add1 (handles, session_handle (session));
+ f = f->next;
+ }
}
/* Instead of removing the segment, test when cleaning up disconnected
fifo_segment_attach_fifo (fs, f, s->thread_index);
segment_manager_segment_reader_unlock (sm);
- f->master_session_index = s->session_index;
+ f->shr->master_session_index = s->session_index;
f->master_thread_index = s->thread_index;
}
u32 session_index, thread_index;
session_t *session;
- session_index = f->master_session_index;
- thread_index = f->master_thread_index;
+ session_index = f->shr->master_session_index;
+ thread_index = f->master_thread_index;
- session = session_get (session_index, thread_index);
- str = format (0, "%U", format_session, session, verbose);
+ session = session_get (session_index, thread_index);
+ str = format (0, "%U", format_session, session, verbose);
- if (verbose)
- s = format (s, "%-40v%-20v%-15u%-10u", str, app_name,
- app_wrk->api_client_index, app_wrk->connects_seg_manager);
- else
- s = format (s, "%-40v%-20v", str, app_name);
+ if (verbose)
+ s = format (s, "%-40v%-20v%-15u%-10u", str, app_name,
+ app_wrk->api_client_index,
+ app_wrk->connects_seg_manager);
+ else
+ s = format (s, "%-40v%-20v", str, app_name);
- vlib_cli_output (vm, "%v", s);
- vec_reset_length (s);
- vec_free (str);
+ vlib_cli_output (vm, "%v", s);
+ vec_reset_length (s);
+ vec_free (str);
- f = f->next;
- }
- vec_free (s);
+ f = f->next;
+ }
+ vec_free (s);
}
}
/* *INDENT-ON* */
int
session_send_io_evt_to_thread (svm_fifo_t * f, session_evt_type_t evt_type)
{
- return session_send_evt_to_thread (&f->master_session_index, 0,
+ return session_send_evt_to_thread (&f->shr->master_session_index, 0,
f->master_thread_index, evt_type);
}
{
segment_manager_t *sm;
sm = segment_manager_get (f->segment_manager);
- ASSERT (f->size >= 4096);
- ASSERT (f->size <= sm->max_fifo_size);
+ ASSERT (f->shr->size >= 4096);
+ ASSERT (f->shr->size <= sm->max_fifo_size);
}
}
}
if (!app)
return -1;
- for (i = 0; i < f->n_subscribers; i++)
+ for (i = 0; i < f->shr->n_subscribers; i++)
{
- app_wrk = application_get_worker (app, f->subscribers[i]);
+ app_wrk = application_get_worker (app, f->shr->subscribers[i]);
if (!app_wrk)
continue;
if (app_worker_lock_and_send_event (app_wrk, s, evt_type))
SESSION_IO_EVT_TX)))
return -1;
- if (PREDICT_FALSE (s->tx_fifo->n_subscribers))
+ if (PREDICT_FALSE (s->tx_fifo->shr->n_subscribers))
return session_notify_subscribers (app_wrk->app_index, s,
s->tx_fifo, SESSION_IO_EVT_TX);
mp = (session_accepted_msg_t *) evt->data;
clib_memset (mp, 0, sizeof (*mp));
mp->context = app->app_index;
- mp->server_rx_fifo = pointer_to_uword (s->rx_fifo);
- mp->server_tx_fifo = pointer_to_uword (s->tx_fifo);
+ mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr);
+ mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr);
mp->segment_handle = session_segment_handle (s);
mp->flags = s->flags;
if (!app)
return;
- for (i = 0; i < f->n_subscribers; i++)
+ for (i = 0; i < f->shr->n_subscribers; i++)
{
- if (!(app_wrk = application_get_worker (app, f->subscribers[i])))
+ if (!(app_wrk = application_get_worker (app, f->shr->subscribers[i])))
continue;
mq_send_session_close_evt (app_wrk, sh, SESSION_CTRL_EVT_DISCONNECTED);
}
session_get_endpoint (s, &mp->lcl, 1 /* is_lcl */ );
- mp->server_rx_fifo = pointer_to_uword (s->rx_fifo);
- mp->server_tx_fifo = pointer_to_uword (s->tx_fifo);
+ mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr);
+ mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr);
mp->segment_handle = session_segment_handle (s);
}
else
mp->lcl.is_ip4 = cct->c_is_ip4;
vpp_mq = session_main_get_vpp_event_queue (s->thread_index);
mp->vpp_event_queue_address = pointer_to_uword (vpp_mq);
- mp->server_rx_fifo = pointer_to_uword (s->rx_fifo);
- mp->server_tx_fifo = pointer_to_uword (s->tx_fifo);
+ mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr);
+ mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr);
mp->segment_handle = session_segment_handle (s);
ss = ct_session_get_peer (s);
- mp->ct_rx_fifo = pointer_to_uword (ss->tx_fifo);
- mp->ct_tx_fifo = pointer_to_uword (ss->rx_fifo);
+ mp->ct_rx_fifo = pointer_to_uword (ss->tx_fifo->shr);
+ mp->ct_tx_fifo = pointer_to_uword (ss->rx_fifo->shr);
mp->ct_segment_handle = session_segment_handle (ss);
}
if (session_transport_service_type (ls) == TRANSPORT_SERVICE_CL)
{
- mp->rx_fifo = pointer_to_uword (ls->rx_fifo);
- mp->tx_fifo = pointer_to_uword (ls->tx_fifo);
+ mp->rx_fifo = pointer_to_uword (ls->rx_fifo->shr);
+ mp->tx_fifo = pointer_to_uword (ls->tx_fifo->shr);
+ mp->segment_handle = session_segment_handle (ls);
}
done:
mp->vpp_thread_index = session_thread_from_handle (new_sh);
vpp_evt_q = session_main_get_vpp_event_queue (mp->vpp_thread_index);
mp->vpp_evt_q = pointer_to_uword (vpp_evt_q);
+ mp->segment_handle = session_segment_handle (s);
svm_msg_q_add_and_unlock (app_mq, msg);
}
return s;
s = format (s, " Rx fifo: %U", format_svm_fifo, ss->rx_fifo, verbose);
- if (verbose > 2 && ss->rx_fifo->has_event)
+ if (verbose > 2 && ss->rx_fifo->shr->has_event)
{
found = session_node_lookup_fifo_event (ss->rx_fifo, e);
s = format (s, " session node event: %s\n",
found ? "found" : "not found");
}
s = format (s, " Tx fifo: %U", format_svm_fifo, ss->tx_fifo, verbose);
- if (verbose > 2 && ss->tx_fifo->has_event)
+ if (verbose > 2 && ss->tx_fifo->shr->has_event)
{
found = session_node_lookup_fifo_event (ss->tx_fifo, e);
s = format (s, " session node event: %s\n",
case SESSION_IO_EVT_BUILTIN_RX:
case SESSION_IO_EVT_BUILTIN_TX:
case SESSION_IO_EVT_TX_FLUSH:
- if (e->session_index == f->master_session_index)
+ if (e->session_index == f->shr->master_session_index)
return 1;
break;
case SESSION_CTRL_EVT_CLOSE:
Code Review / vpp.git / commitdiff