+ vep_session->vpp_handle = ~0;
+
+ vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_sh);
+ VDBG (0, "VCL<%d>: Created vep_idx %u / sid %u!",
+ getpid (), vep_session->session_index, vep_session->session_index);
+
+ return vcl_session_handle (vep_session);
+}
+
+int
+vppcom_epoll_ctl (uint32_t vep_handle, int op, uint32_t session_handle,
+ struct epoll_event *event)
+{
+ vcl_worker_t *wrk = vcl_worker_get_current ();
+ vcl_session_t *vep_session;
+ vcl_session_t *session;
+ int rv = VPPCOM_OK;
+
+ if (vep_handle == session_handle)
+ {
+ clib_warning ("VCL<%d>: ERROR: vep_idx == session_index (%u)!",
+ getpid (), vep_handle);
+ return VPPCOM_EINVAL;
+ }
+
+ vep_session = vcl_session_get_w_handle (wrk, vep_handle);
+ if (PREDICT_FALSE (!vep_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: Invalid vep_idx (%u)!", vep_handle);
+ return VPPCOM_EBADFD;
+ }
+ if (PREDICT_FALSE (!vep_session->is_vep))
+ {
+ clib_warning ("VCL<%d>: ERROR: vep_idx (%u) is not a vep!",
+ getpid (), vep_handle);
+ return VPPCOM_EINVAL;
+ }
+
+ ASSERT (vep_session->vep.vep_sh == ~0);
+ ASSERT (vep_session->vep.prev_sh == ~0);
+
+ session = vcl_session_get_w_handle (wrk, session_handle);
+ if (PREDICT_FALSE (!session))
+ {
+ VDBG (0, "VCL<%d>: ERROR: Invalid session_handle (%u)!",
+ getpid (), session_handle);
+ return VPPCOM_EBADFD;
+ }
+ if (PREDICT_FALSE (session->is_vep))
+ {
+ clib_warning ("ERROR: session_handle (%u) is a vep!", vep_handle);
+ return VPPCOM_EINVAL;
+ }
+
+ switch (op)
+ {
+ case EPOLL_CTL_ADD:
+ if (PREDICT_FALSE (!event))
+ {
+ clib_warning ("VCL<%d>: ERROR: EPOLL_CTL_ADD: NULL pointer to "
+ "epoll_event structure!", getpid ());
+ return VPPCOM_EINVAL;
+ }
+ if (vep_session->vep.next_sh != ~0)
+ {
+ vcl_session_t *next_session;
+ next_session = vcl_session_get_w_handle (wrk,
+ vep_session->vep.next_sh);
+ if (PREDICT_FALSE (!next_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: EPOLL_CTL_ADD: Invalid "
+ "vep.next_sid (%u) on vep_idx (%u)!",
+ getpid (), vep_session->vep.next_sh, vep_handle);
+ return VPPCOM_EBADFD;
+ }
+ ASSERT (next_session->vep.prev_sh == vep_handle);
+ next_session->vep.prev_sh = session_handle;
+ }
+ session->vep.next_sh = vep_session->vep.next_sh;
+ session->vep.prev_sh = vep_handle;
+ session->vep.vep_sh = vep_handle;
+ session->vep.et_mask = VEP_DEFAULT_ET_MASK;
+ session->vep.ev = *event;
+ session->is_vep = 0;
+ session->is_vep_session = 1;
+ vep_session->vep.next_sh = session_handle;
+
+ VDBG (1, "VCL<%d>: EPOLL_CTL_ADD: vep_idx %u, sid %u, events 0x%x, "
+ "data 0x%llx!", getpid (), vep_handle, session_handle,
+ event->events, event->data.u64);
+ vcl_evt (VCL_EVT_EPOLL_CTLADD, session, event->events, event->data.u64);
+ break;
+
+ case EPOLL_CTL_MOD:
+ if (PREDICT_FALSE (!event))
+ {
+ clib_warning ("VCL<%d>: ERROR: EPOLL_CTL_MOD: NULL pointer to "
+ "epoll_event structure!", getpid ());
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ else if (PREDICT_FALSE (!session->is_vep_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: sid %u EPOLL_CTL_MOD: "
+ "not a vep session!", getpid (), session_handle);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ else if (PREDICT_FALSE (session->vep.vep_sh != vep_handle))
+ {
+ clib_warning ("VCL<%d>: ERROR: sid %u EPOLL_CTL_MOD: "
+ "vep_idx (%u) != vep_idx (%u)!",
+ getpid (), session_handle,
+ session->vep.vep_sh, vep_handle);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ session->vep.et_mask = VEP_DEFAULT_ET_MASK;
+ session->vep.ev = *event;
+ VDBG (1, "VCL<%d>: EPOLL_CTL_MOD: vep_idx %u, sid %u, events 0x%x,"
+ " data 0x%llx!", getpid (), vep_handle, session_handle,
+ event->events, event->data.u64);
+ break;
+
+ case EPOLL_CTL_DEL:
+ if (PREDICT_FALSE (!session->is_vep_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: sid %u EPOLL_CTL_DEL: "
+ "not a vep session!", getpid (), session_handle);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ else if (PREDICT_FALSE (session->vep.vep_sh != vep_handle))
+ {
+ clib_warning ("VCL<%d>: ERROR: sid %u EPOLL_CTL_DEL: "
+ "vep_idx (%u) != vep_idx (%u)!",
+ getpid (), session_handle,
+ session->vep.vep_sh, vep_handle);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+
+ vep_session->wait_cont_idx =
+ (vep_session->wait_cont_idx == session_handle) ?
+ session->vep.next_sh : vep_session->wait_cont_idx;
+
+ if (session->vep.prev_sh == vep_handle)
+ vep_session->vep.next_sh = session->vep.next_sh;
+ else
+ {
+ vcl_session_t *prev_session;
+ prev_session = vcl_session_get_w_handle (wrk, session->vep.prev_sh);
+ if (PREDICT_FALSE (!prev_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: EPOLL_CTL_DEL: Invalid "
+ "vep.prev_sid (%u) on sid (%u)!",
+ getpid (), session->vep.prev_sh, session_handle);
+ return VPPCOM_EBADFD;
+ }
+ ASSERT (prev_session->vep.next_sh == session_handle);
+ prev_session->vep.next_sh = session->vep.next_sh;
+ }
+ if (session->vep.next_sh != ~0)
+ {
+ vcl_session_t *next_session;
+ next_session = vcl_session_get_w_handle (wrk, session->vep.next_sh);
+ if (PREDICT_FALSE (!next_session))
+ {
+ clib_warning ("VCL<%d>: ERROR: EPOLL_CTL_DEL: Invalid "
+ "vep.next_sid (%u) on sid (%u)!",
+ getpid (), session->vep.next_sh, session_handle);
+ return VPPCOM_EBADFD;
+ }
+ ASSERT (next_session->vep.prev_sh == session_handle);
+ next_session->vep.prev_sh = session->vep.prev_sh;
+ }
+
+ memset (&session->vep, 0, sizeof (session->vep));
+ session->vep.next_sh = ~0;
+ session->vep.prev_sh = ~0;
+ session->vep.vep_sh = ~0;
+ session->is_vep_session = 0;
+ VDBG (1, "VCL<%d>: EPOLL_CTL_DEL: vep_idx %u, sid %u!",
+ getpid (), vep_handle, session_handle);
+ vcl_evt (VCL_EVT_EPOLL_CTLDEL, session, vep_sh);
+ break;
+
+ default:
+ clib_warning ("VCL<%d>: ERROR: Invalid operation (%d)!", getpid (), op);
+ rv = VPPCOM_EINVAL;
+ }
+
+ vep_verify_epoll_chain (wrk, vep_handle);
+
+done:
+ return rv;
+}
+
+static inline void
+vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
+ struct epoll_event *events, u32 * num_ev)
+{
+ session_disconnected_msg_t *disconnected_msg;
+ session_connected_msg_t *connected_msg;
+ session_accepted_msg_t *accepted_msg;
+ u64 session_evt_data = ~0, handle;
+ u32 sid = ~0, session_events;
+ vcl_session_msg_t *vcl_msg;
+ vcl_session_t *session;
+ u8 add_event = 0;
+
+ switch (e->event_type)
+ {
+ case FIFO_EVENT_APP_RX:
+ ASSERT (e->fifo->client_thread_index == vcl_get_worker_index ());
+ vcl_fifo_rx_evt_valid_or_break (e->fifo);
+ sid = e->fifo->client_session_index;
+ session = vcl_session_get (wrk, sid);
+ session_events = session->vep.ev.events;
+ if (!(EPOLLIN & session->vep.ev.events) || session->has_rx_evt)
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLIN;
+ session_evt_data = session->vep.ev.data.u64;
+ session->has_rx_evt = 1;
+ break;
+ case FIFO_EVENT_APP_TX:
+ sid = e->fifo->client_session_index;
+ session = vcl_session_get (wrk, sid);
+ session_events = session->vep.ev.events;
+ if (!(EPOLLOUT & session_events))
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLOUT;
+ session_evt_data = session->vep.ev.data.u64;
+ break;
+ case SESSION_IO_EVT_CT_TX:
+ vcl_fifo_rx_evt_valid_or_break (e->fifo);
+ session = vcl_ct_session_get_from_fifo (wrk, e->fifo, 0);
+ sid = session->session_index;
+ session_events = session->vep.ev.events;
+ if (!(EPOLLIN & session->vep.ev.events) || session->has_rx_evt)
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLIN;
+ session_evt_data = session->vep.ev.data.u64;
+ session->has_rx_evt = 1;
+ break;
+ case SESSION_IO_EVT_CT_RX:
+ session = vcl_ct_session_get_from_fifo (wrk, e->fifo, 1);
+ sid = session->session_index;
+ session_events = session->vep.ev.events;
+ if (!(EPOLLOUT & session_events))
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLOUT;
+ session_evt_data = session->vep.ev.data.u64;
+ break;
+ case SESSION_CTRL_EVT_ACCEPTED:
+ accepted_msg = (session_accepted_msg_t *) e->data;
+ handle = accepted_msg->listener_handle;
+ session = vcl_session_table_lookup_listener (wrk, handle);
+ if (!session)
+ {
+ clib_warning ("VCL<%d>: ERROR: couldn't find listen session:"
+ "listener handle %llx", getpid (), handle);
+ break;
+ }
+
+ clib_fifo_add2 (session->accept_evts_fifo, vcl_msg);
+ vcl_msg->accepted_msg = *accepted_msg;
+ session_events = session->vep.ev.events;
+ if (!(EPOLLIN & session_events))
+ break;
+
+ add_event = 1;
+ events[*num_ev].events |= EPOLLIN;
+ session_evt_data = session->vep.ev.data.u64;
+ break;
+ case SESSION_CTRL_EVT_CONNECTED:
+ connected_msg = (session_connected_msg_t *) e->data;
+ vcl_session_connected_handler (wrk, connected_msg);
+ /* Generate EPOLLOUT because there's no connected event */
+ sid = vcl_session_index_from_vpp_handle (wrk, connected_msg->handle);
+ session = vcl_session_get (wrk, sid);
+ session_events = session->vep.ev.events;
+ if (EPOLLOUT & session_events)
+ {
+ add_event = 1;
+ events[*num_ev].events |= EPOLLOUT;
+ session_evt_data = session->vep.ev.data.u64;
+ }
+ break;
+ case SESSION_CTRL_EVT_DISCONNECTED:
+ disconnected_msg = (session_disconnected_msg_t *) e->data;
+ sid = vcl_session_index_from_vpp_handle (wrk, disconnected_msg->handle);
+ if (!(session = vcl_session_get (wrk, sid)))
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
+ session_evt_data = session->vep.ev.data.u64;
+ session_events = session->vep.ev.events;
+ break;
+ case SESSION_CTRL_EVT_RESET:
+ sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
+ if (!(session = vcl_session_get (wrk, sid)))
+ break;
+ add_event = 1;
+ events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
+ session_evt_data = session->vep.ev.data.u64;
+ session_events = session->vep.ev.events;
+ break;
+ default:
+ VDBG (0, "unhandled: %u", e->event_type);
+ break;
+ }
+
+ if (add_event)
+ {
+ events[*num_ev].data.u64 = session_evt_data;
+ if (EPOLLONESHOT & session_events)
+ {
+ session = vcl_session_get (wrk, sid);
+ session->vep.ev.events = 0;
+ }
+ *num_ev += 1;
+ }
+}
+
+static int
+vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
+ struct epoll_event *events, u32 maxevents,
+ double wait_for_time, u32 * num_ev)
+{
+ svm_msg_q_msg_t *msg;
+ session_event_t *e;
+ int i;
+
+ if (vec_len (wrk->mq_msg_vector) && svm_msg_q_is_empty (mq))
+ goto handle_dequeued;
+
+ svm_msg_q_lock (mq);
+ if (svm_msg_q_is_empty (mq))
+ {
+ if (!wait_for_time)
+ {
+ svm_msg_q_unlock (mq);
+ return 0;
+ }
+ else if (wait_for_time < 0)
+ {
+ svm_msg_q_wait (mq);
+ }
+ else
+ {
+ if (svm_msg_q_timedwait (mq, wait_for_time / 1e3))
+ {
+ svm_msg_q_unlock (mq);
+ return 0;
+ }
+ }
+ }
+ vcl_mq_dequeue_batch (wrk, mq);
+ svm_msg_q_unlock (mq);
+
+handle_dequeued:
+ for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
+ {
+ msg = vec_elt_at_index (wrk->mq_msg_vector, i);
+ e = svm_msg_q_msg_data (mq, msg);
+ if (*num_ev < maxevents)
+ vcl_epoll_wait_handle_mq_event (wrk, e, events, num_ev);
+ else
+ vec_add1 (wrk->unhandled_evts_vector, *e);
+ svm_msg_q_free_msg (mq, msg);
+ }
+ vec_reset_length (wrk->mq_msg_vector);
+
+ return *num_ev;
+}
+
+static int
+vppcom_epoll_wait_condvar (vcl_worker_t * wrk, struct epoll_event *events,
+ int maxevents, u32 n_evts, double wait_for_time)
+{
+ vcl_cut_through_registration_t *cr;
+ double total_wait = 0, wait_slice;
+ int rv;
+
+ wait_for_time = (wait_for_time == -1) ? (double) 10e9 : wait_for_time;
+ wait_slice = wrk->cut_through_registrations ? 10e-6 : wait_for_time;
+
+ do
+ {
+ vcl_ct_registration_lock (wrk);
+ /* *INDENT-OFF* */
+ pool_foreach (cr, wrk->cut_through_registrations, ({
+ vcl_epoll_wait_handle_mq (wrk, cr->mq, events, maxevents, 0, &n_evts);
+ }));
+ /* *INDENT-ON* */
+ vcl_ct_registration_unlock (wrk);
+
+ rv = vcl_epoll_wait_handle_mq (wrk, wrk->app_event_queue, events,
+ maxevents, n_evts ? 0 : wait_slice,
+ &n_evts);
+ if (rv)
+ total_wait += wait_slice;
+ if (n_evts)
+ return n_evts;
+ }
+ while (total_wait < wait_for_time);
+ return n_evts;
+}
+
+static int
+vppcom_epoll_wait_eventfd (vcl_worker_t * wrk, struct epoll_event *events,
+ int maxevents, u32 n_evts, double wait_for_time)
+{
+ vcl_mq_evt_conn_t *mqc;
+ int __clib_unused n_read;
+ int n_mq_evts, i;
+ u64 buf;
+
+ vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
+again:
+ n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
+ vec_len (wrk->mq_events), wait_for_time);
+ for (i = 0; i < n_mq_evts; i++)
+ {
+ mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
+ n_read = read (mqc->mq_fd, &buf, sizeof (buf));
+ vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0, &n_evts);
+ }
+ if (!n_evts && n_mq_evts > 0)
+ goto again;
+
+ return (int) n_evts;
+}
+
+int
+vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
+ int maxevents, double wait_for_time)
+{
+ vcl_worker_t *wrk = vcl_worker_get_current ();
+ vcl_session_t *vep_session;
+ u32 n_evts = 0;
+ int i;
+
+ if (PREDICT_FALSE (maxevents <= 0))
+ {
+ clib_warning ("VCL<%d>: ERROR: Invalid maxevents (%d)!",
+ getpid (), maxevents);
+ return VPPCOM_EINVAL;
+ }
+
+ vep_session = vcl_session_get_w_handle (wrk, vep_handle);
+ if (!vep_session)
+ return VPPCOM_EBADFD;
+
+ if (PREDICT_FALSE (!vep_session->is_vep))
+ {
+ clib_warning ("VCL<%d>: ERROR: vep_idx (%u) is not a vep!",
+ getpid (), vep_handle);
+ return VPPCOM_EINVAL;
+ }
+
+ memset (events, 0, sizeof (*events) * maxevents);
+
+ if (vec_len (wrk->unhandled_evts_vector))
+ {
+ for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
+ {
+ vcl_epoll_wait_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i],
+ events, &n_evts);
+ if (n_evts == maxevents)
+ {
+ i += 1;
+ break;
+ }
+ }
+
+ vec_delete (wrk->unhandled_evts_vector, i, 0);
+ }