+static inline void
+vep_verify_epoll_chain (u32 vep_idx)
+{
+ session_t *session;
+ vppcom_epoll_t *vep;
+ int rv;
+ u32 sid;
+
+ if (VPPCOM_DEBUG < 1)
+ return;
+
+ /* Assumes caller has acquired spinlock: vcm->sessions_lockp */
+ rv = vppcom_session_at_index (vep_idx, &session);
+ if (PREDICT_FALSE (rv))
+ {
+ clib_warning ("ERROR: Invalid vep_idx (%u)!", vep_idx);
+ goto done;
+ }
+ if (PREDICT_FALSE (!session->is_vep))
+ {
+ clib_warning ("ERROR: vep_idx (%u) is not a vep!", vep_idx);
+ goto done;
+ }
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("vep_idx (%u): Dumping epoll chain\n"
+ "{\n"
+ " is_vep = %u\n"
+ " is_vep_session = %u\n"
+ " wait_cont_idx = 0x%x (%u)\n"
+ "}\n",
+ vep_idx, session->is_vep, session->is_vep_session,
+ session->wait_cont_idx, session->wait_cont_idx);
+ do
+ {
+ vep = &session->vep;
+ if (session->is_vep_session)
+ {
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("vep_idx[%u]: sid 0x%x (%u)\n"
+ "{\n"
+ " next_sid = 0x%x (%u)\n"
+ " prev_sid = 0x%x (%u)\n"
+ " vep_idx = 0x%x (%u)\n"
+ " ev.events = 0x%x\n"
+ " ev.data.u64 = 0x%llx\n"
+ " et_mask = 0x%x\n"
+ "}\n",
+ vep_idx, sid, sid,
+ vep->next_sid, vep->next_sid,
+ vep->prev_sid, vep->prev_sid,
+ vep->vep_idx, vep->vep_idx,
+ vep->ev.events, vep->ev.data.u64, vep->et_mask);
+ }
+ sid = vep->next_sid;
+ if (sid != ~0)
+ {
+ rv = vppcom_session_at_index (sid, &session);
+ if (PREDICT_FALSE (rv))
+ {
+ clib_warning ("ERROR: Invalid sid (%u)!", sid);
+ goto done;
+ }
+ if (PREDICT_FALSE (session->is_vep))
+ clib_warning ("ERROR: sid (%u) is a vep!", vep_idx);
+ else if (PREDICT_FALSE (!session->is_vep_session))
+ {
+ clib_warning ("ERROR: session (%u) is not a vep session!", sid);
+ goto done;
+ }
+ if (PREDICT_FALSE (session->vep.vep_idx != vep_idx))
+ clib_warning ("ERROR: session (%u) vep_idx (%u) != "
+ "vep_idx (%u)!",
+ sid, session->vep.vep_idx, vep_idx);
+ }
+ }
+ while (sid != ~0);
+
+done:
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("vep_idx (%u): Dump complete!", vep_idx);
+}
+
+int
+vppcom_epoll_create (void)
+{
+ vppcom_main_t *vcm = &vppcom_main;
+ session_t *vep_session;
+ u32 vep_idx;
+
+ clib_spinlock_lock (&vcm->sessions_lockp);
+ pool_get (vcm->sessions, vep_session);
+ memset (vep_session, 0, sizeof (*vep_session));
+ vep_idx = vep_session - vcm->sessions;
+
+ vep_session->is_vep = 1;
+ vep_session->vep.vep_idx = ~0;
+ vep_session->vep.next_sid = ~0;
+ vep_session->vep.prev_sid = ~0;
+ vep_session->wait_cont_idx = ~0;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("Created vep_idx %u!", vep_idx);
+
+ return (vep_idx);
+}
+
+int
+vppcom_epoll_ctl (uint32_t vep_idx, int op, uint32_t session_index,
+ struct epoll_event *event)
+{
+ vppcom_main_t *vcm = &vppcom_main;
+ session_t *vep_session;
+ session_t *session;
+ int rv;
+
+ if (vep_idx == session_index)
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: vep_idx == session_index (%u)!", vep_idx);
+ return VPPCOM_EINVAL;
+ }
+
+ clib_spinlock_lock (&vcm->sessions_lockp);
+ rv = vppcom_session_at_index (vep_idx, &vep_session);
+ if (PREDICT_FALSE (rv))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: Invalid vep_idx (%u)!", vep_idx);
+ goto done;
+ }
+ if (PREDICT_FALSE (!vep_session->is_vep))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: vep_idx (%u) is not a vep!", vep_idx);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+
+ ASSERT (vep_session->vep.vep_idx == ~0);
+ ASSERT (vep_session->vep.prev_sid == ~0);
+
+ rv = vppcom_session_at_index (session_index, &session);
+ if (PREDICT_FALSE (rv))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: Invalid session_index (%u)!", session_index);
+ goto done;
+ }
+ if (PREDICT_FALSE (session->is_vep))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: session_index (%u) is a vep!", vep_idx);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+
+ switch (op)
+ {
+ case EPOLL_CTL_ADD:
+ if (PREDICT_FALSE (!event))
+ {
+ clib_warning ("NULL pointer to epoll_event structure!");
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ if (vep_session->vep.next_sid != ~0)
+ {
+ session_t *next_session;
+ rv = vppcom_session_at_index (vep_session->vep.next_sid,
+ &next_session);
+ if (PREDICT_FALSE (rv))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("EPOLL_CTL_ADD: Invalid vep.next_sid (%u) on"
+ " vep_idx (%u)!", vep_session->vep.next_sid,
+ vep_idx);
+ goto done;
+ }
+ ASSERT (next_session->vep.prev_sid == vep_idx);
+ next_session->vep.prev_sid = session_index;
+ }
+ session->vep.next_sid = vep_session->vep.next_sid;
+ session->vep.prev_sid = vep_idx;
+ session->vep.vep_idx = vep_idx;
+ session->vep.et_mask = VEP_DEFAULT_ET_MASK;
+ session->vep.ev = *event;
+ session->is_vep_session = 1;
+ vep_session->vep.next_sid = session_index;
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("EPOLL_CTL_ADD: vep_idx %u, sid %u, events 0x%x,"
+ " data 0x%llx!", vep_idx, session_index,
+ event->events, event->data.u64);
+ break;
+
+ case EPOLL_CTL_MOD:
+ if (PREDICT_FALSE (!event))
+ {
+ clib_warning ("NULL pointer to epoll_event structure!");
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ if (PREDICT_FALSE (!session->is_vep_session &&
+ (session->vep.vep_idx != vep_idx)))
+ {
+ if (VPPCOM_DEBUG > 0)
+ {
+ if (!session->is_vep_session)
+ clib_warning ("EPOLL_CTL_MOD: session (%u) is not "
+ "a vep session!", session_index);
+ else
+ clib_warning ("EPOLL_CTL_MOD: session (%u) vep_idx (%u) != "
+ "vep_idx (%u)!", session_index,
+ session->vep.vep_idx, vep_idx);
+ }
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ session->vep.et_mask = VEP_DEFAULT_ET_MASK;
+ session->vep.ev = *event;
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("EPOLL_CTL_MOD: vep_idx %u, sid %u, events 0x%x,"
+ " data 0x%llx!", vep_idx, session_index,
+ event->events, event->data.u64);
+ break;
+
+ case EPOLL_CTL_DEL:
+ if (PREDICT_FALSE (!session->is_vep_session &&
+ (session->vep.vep_idx != vep_idx)))
+ {
+ if (VPPCOM_DEBUG > 0)
+ {
+ if (!session->is_vep_session)
+ clib_warning ("EPOLL_CTL_DEL: session (%u) is not "
+ "a vep session!", session_index);
+ else
+ clib_warning ("EPOLL_CTL_DEL: session (%u) vep_idx (%u) != "
+ "vep_idx (%u)!", session_index,
+ session->vep.vep_idx, vep_idx);
+ }
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+
+ vep_session->wait_cont_idx =
+ (vep_session->wait_cont_idx == session_index) ?
+ session->vep.next_sid : vep_session->wait_cont_idx;
+
+ if (session->vep.prev_sid == vep_idx)
+ vep_session->vep.next_sid = session->vep.next_sid;
+ else
+ {
+ session_t *prev_session;
+ rv = vppcom_session_at_index (session->vep.prev_sid, &prev_session);
+ if (PREDICT_FALSE (rv))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("EPOLL_CTL_DEL: Invalid vep.prev_sid (%u) on"
+ " sid (%u)!", session->vep.prev_sid,
+ session_index);
+ goto done;
+ }
+ ASSERT (prev_session->vep.next_sid == session_index);
+ prev_session->vep.next_sid = session->vep.next_sid;
+ }
+ if (session->vep.next_sid != ~0)
+ {
+ session_t *next_session;
+ rv = vppcom_session_at_index (session->vep.next_sid, &next_session);
+ if (PREDICT_FALSE (rv))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("EPOLL_CTL_DEL: Invalid vep.next_sid (%u) on"
+ " sid (%u)!", session->vep.next_sid,
+ session_index);
+ goto done;
+ }
+ ASSERT (next_session->vep.prev_sid == session_index);
+ next_session->vep.prev_sid = session->vep.prev_sid;
+ }
+
+ memset (&session->vep, 0, sizeof (session->vep));
+ session->vep.next_sid = ~0;
+ session->vep.prev_sid = ~0;
+ session->vep.vep_idx = ~0;
+ session->is_vep_session = 0;
+ if (VPPCOM_DEBUG > 1)
+ clib_warning ("EPOLL_CTL_DEL: vep_idx %u, sid %u!", vep_idx,
+ session_index);
+ break;
+
+ default:
+ clib_warning ("Invalid operation (%d)!", op);
+ rv = VPPCOM_EINVAL;
+ }
+
+ vep_verify_epoll_chain (vep_idx);
+
+done:
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ return rv;
+}
+
+#define VCL_LOCK_AND_GET_SESSION(I, S) \
+do { \
+ vppcom_main_t *vcm = &vppcom_main; \
+ \
+ clib_spinlock_lock (&vcm->sessions_lockp); \
+ rv = vppcom_session_at_index (I, S); \
+ if (PREDICT_FALSE (rv)) \
+ { \
+ clib_spinlock_unlock (&vcm->sessions_lockp); \
+ \
+ if (VPPCOM_DEBUG > 0) \
+ clib_warning ("ERROR: Invalid ##I (%u)!", I); \
+ \
+ goto done; \
+ } \
+} while (0)
+
+int
+vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events,
+ int maxevents, double wait_for_time)
+{
+ vppcom_main_t *vcm = &vppcom_main;
+ session_t *vep_session;
+ int rv;
+ f64 timeout = clib_time_now (&vcm->clib_time) + wait_for_time;
+ int num_ev = 0;
+ u32 vep_next_sid, wait_cont_idx;
+ u8 is_vep;
+
+ if (PREDICT_FALSE (maxevents <= 0))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: Invalid maxevents (%d)!", maxevents);
+ return VPPCOM_EINVAL;
+ }
+ if (PREDICT_FALSE (wait_for_time < 0))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: Invalid wait_for_time (%f)!", wait_for_time);
+ return VPPCOM_EINVAL;
+ }
+ memset (events, 0, sizeof (*events) * maxevents);
+
+ VCL_LOCK_AND_GET_SESSION (vep_idx, &vep_session);
+ vep_next_sid = vep_session->vep.next_sid;
+ is_vep = vep_session->is_vep;
+ wait_cont_idx = vep_session->wait_cont_idx;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+
+ if (PREDICT_FALSE (!is_vep))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: vep_idx (%u) is not a vep!", vep_idx);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ if ((VPPCOM_DEBUG > 0) && (PREDICT_FALSE (vep_next_sid == ~0)))
+ {
+ clib_warning ("WARNING: vep_idx (%u) is empty!", vep_idx);
+ goto done;
+ }
+
+ do
+ {
+ u32 sid;
+ u32 next_sid = ~0;
+ session_t *session;
+
+ for (sid = (wait_cont_idx == ~0) ? vep_next_sid : wait_cont_idx;
+ sid != ~0; sid = next_sid)
+ {
+ u32 session_events, et_mask, clear_et_mask, session_vep_idx;
+ u8 add_event, is_vep_session;
+ int ready;
+ u64 session_ev_data;
+
+ VCL_LOCK_AND_GET_SESSION (sid, &session);
+ next_sid = session->vep.next_sid;
+ session_events = session->vep.ev.events;
+ et_mask = session->vep.et_mask;
+ is_vep = session->is_vep;
+ is_vep_session = session->is_vep_session;
+ session_vep_idx = session->vep.vep_idx;
+ session_ev_data = session->vep.ev.data.u64;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+
+ if (PREDICT_FALSE (is_vep))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("ERROR: sid (%u) is a vep!", vep_idx);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ if (PREDICT_FALSE (!is_vep_session))
+ {
+ if (VPPCOM_DEBUG > 0)
+ clib_warning ("EPOLL_CTL_MOD: session (%u) is not "
+ "a vep session!", sid);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+ if (PREDICT_FALSE (session_vep_idx != vep_idx))
+ {
+ clib_warning ("EPOLL_CTL_MOD: session (%u) "
+ "vep_idx (%u) != vep_idx (%u)!",
+ sid, session->vep.vep_idx, vep_idx);
+ rv = VPPCOM_EINVAL;
+ goto done;
+ }
+
+ add_event = clear_et_mask = 0;
+
+ if ((EPOLLIN & session_events) && (EPOLLIN & et_mask))
+ {
+ VCL_LOCK_AND_GET_SESSION (sid, &session);
+ ready = vppcom_session_read_ready (session, sid);
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ if (ready > 0)
+ {
+ add_event = 1;
+ events[num_ev].events |= EPOLLIN;
+ if (EPOLLET & session_events)
+ clear_et_mask |= EPOLLIN;
+ }
+ else if (ready < 0)
+ {
+ add_event = 1;
+ switch (ready)
+ {
+ case VPPCOM_ECONNRESET:
+ events[num_ev].events |= EPOLLHUP | EPOLLRDHUP;
+ break;
+
+ default:
+ events[num_ev].events |= EPOLLERR;
+ break;
+ }
+ }
+ }
+
+ if ((EPOLLOUT & session_events) && (EPOLLOUT & et_mask))
+ {
+ VCL_LOCK_AND_GET_SESSION (sid, &session);
+ ready = vppcom_session_write_ready (session, sid);
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ if (ready > 0)
+ {
+ add_event = 1;
+ events[num_ev].events |= EPOLLOUT;
+ if (EPOLLET & session_events)
+ clear_et_mask |= EPOLLOUT;
+ }
+ else if (ready < 0)
+ {
+ add_event = 1;
+ switch (ready)
+ {
+ case VPPCOM_ECONNRESET:
+ events[num_ev].events |= EPOLLHUP;
+ break;
+
+ default:
+ events[num_ev].events |= EPOLLERR;
+ break;
+ }
+ }
+ }
+
+ if (add_event)
+ {
+ events[num_ev].data.u64 = session_ev_data;
+ if (EPOLLONESHOT & session_events)
+ {
+ VCL_LOCK_AND_GET_SESSION (sid, &session);
+ session->vep.ev.events = 0;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ }
+ num_ev++;
+ if (num_ev == maxevents)
+ {
+ VCL_LOCK_AND_GET_SESSION (vep_idx, &vep_session);
+ vep_session->wait_cont_idx = next_sid;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ goto done;
+ }
+ }
+ if (wait_cont_idx != ~0)
+ {
+ if (next_sid == ~0)
+ next_sid = vep_next_sid;
+ else if (next_sid == wait_cont_idx)
+ next_sid = ~0;
+ }
+ }
+ }
+ while ((num_ev == 0) && (clib_time_now (&vcm->clib_time) <= timeout));
+
+ if (wait_cont_idx != ~0)
+ {
+ VCL_LOCK_AND_GET_SESSION (vep_idx, &vep_session);
+ vep_session->wait_cont_idx = ~0;
+ clib_spinlock_unlock (&vcm->sessions_lockp);
+ }
+done:
+ return (rv != VPPCOM_OK) ? rv : num_ev;
+}
+
+ /*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */