session tcp: handle rxt and acks as custom events
[vpp.git] / src / vnet / session / session_node.c
index 7855bf8..1bb5cb2 100644 (file)
 #include <vnet/session/session_debug.h>
 #include <svm/queue.h>
 
+static void session_mq_accepted_reply_handler (void *data);
+
+static void
+accepted_notify_cb (void *data, u32 data_len)
+{
+  session_mq_accepted_reply_handler (data);
+}
+
 static void
 session_mq_accepted_reply_handler (void *data)
 {
@@ -32,7 +40,6 @@ session_mq_accepted_reply_handler (void *data)
   vnet_disconnect_args_t _a = { 0 }, *a = &_a;
   session_state_t old_state;
   app_worker_t *app_wrk;
-  local_session_t *ls;
   session_t *s;
 
   /* Server isn't interested, kill the session */
@@ -44,48 +51,45 @@ session_mq_accepted_reply_handler (void *data)
       return;
     }
 
-  if (session_handle_is_local (mp->handle))
+  /* Mail this back from the main thread. We're not polling in main
+   * thread so we're using other workers for notifications. */
+  if (vlib_num_workers () && vlib_get_thread_index () != 0
+      && session_thread_from_handle (mp->handle) == 0)
     {
-      ls = app_worker_get_local_session_from_handle (mp->handle);
-      if (!ls)
-       {
-         clib_warning ("unknown local handle 0x%lx", mp->handle);
-         return;
-       }
-      app_wrk = app_worker_get (ls->app_wrk_index);
-      if (app_wrk->app_index != mp->context)
-       {
-         clib_warning ("server %u doesn't own local handle 0x%lx",
-                       mp->context, mp->handle);
-         return;
-       }
-      if (app_worker_local_session_connect_notify (ls))
+      vl_api_rpc_call_main_thread (accepted_notify_cb, data,
+                                  sizeof (session_accepted_reply_msg_t));
+      return;
+    }
+
+  s = session_get_from_handle_if_valid (mp->handle);
+  if (!s)
+    return;
+
+  app_wrk = app_worker_get (s->app_wrk_index);
+  if (app_wrk->app_index != mp->context)
+    {
+      clib_warning ("app doesn't own session");
+      return;
+    }
+
+  if (!session_has_transport (s))
+    {
+      s->session_state = SESSION_STATE_READY;
+      if (ct_session_connect_notify (s))
        return;
-      ls->session_state = SESSION_STATE_READY;
     }
   else
     {
-      s = session_get_from_handle_if_valid (mp->handle);
-      if (!s)
-       return;
-
-      app_wrk = app_worker_get (s->app_wrk_index);
-      if (app_wrk->app_index != mp->context)
-       {
-         clib_warning ("app doesn't own session");
-         return;
-       }
-
       old_state = s->session_state;
       s->session_state = SESSION_STATE_READY;
-      if (!svm_fifo_is_empty (s->rx_fifo))
-       app_worker_lock_and_send_event (app_wrk, s, FIFO_EVENT_APP_RX);
+
+      if (!svm_fifo_is_empty_prod (s->rx_fifo))
+       app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX);
 
       /* Closed while waiting for app to reply. Resend disconnect */
       if (old_state >= SESSION_STATE_TRANSPORT_CLOSING)
        {
-         application_t *app = application_get (app_wrk->app_index);
-         app->cb_fns.session_disconnect_callback (s);
+         app_worker_close_notify (app_wrk, s);
          s->session_state = old_state;
          return;
        }
@@ -279,13 +283,13 @@ session_mq_worker_update_handler (void *data)
    * Retransmit messages that may have been lost
    */
   if (s->tx_fifo && !svm_fifo_is_empty (s->tx_fifo))
-    session_send_io_evt_to_thread (s->tx_fifo, FIFO_EVENT_APP_TX);
+    session_send_io_evt_to_thread (s->tx_fifo, SESSION_IO_EVT_TX);
 
   if (s->rx_fifo && !svm_fifo_is_empty (s->rx_fifo))
-    app_worker_lock_and_send_event (app_wrk, s, FIFO_EVENT_APP_RX);
+    app_worker_lock_and_send_event (app_wrk, s, SESSION_IO_EVT_RX);
 
   if (s->session_state >= SESSION_STATE_TRANSPORT_CLOSING)
-    app->cb_fns.session_disconnect_callback (s);
+    app_worker_close_notify (app_wrk, s);
 }
 
 vlib_node_registration_t session_queue_node;
@@ -361,11 +365,11 @@ session_tx_fifo_chain_tail (vlib_main_t * vm, session_tx_context_t * ctx,
 {
   vlib_buffer_t *chain_b, *prev_b;
   u32 chain_bi0, to_deq, left_from_seg;
-  session_manager_worker_t *wrk;
+  session_worker_t *wrk;
   u16 len_to_deq, n_bytes_read;
   u8 *data, j;
 
-  wrk = session_manager_get_worker (ctx->s->thread_index);
+  wrk = session_main_get_worker (ctx->s->thread_index);
   b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
   b->total_length_not_including_first_buffer = 0;
 
@@ -391,7 +395,8 @@ session_tx_fifo_chain_tail (vlib_main_t * vm, session_tx_context_t * ctx,
        }
       else
        {
-         if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
+         if (ctx->transport_vft->transport_options.tx_type ==
+             TRANSPORT_TX_DGRAM)
            {
              svm_fifo_t *f = ctx->s->tx_fifo;
              session_dgram_hdr_t *hdr = &ctx->hdr;
@@ -410,8 +415,8 @@ session_tx_fifo_chain_tail (vlib_main_t * vm, session_tx_context_t * ctx,
                }
            }
          else
-           n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo,
-                                                   len_to_deq, data);
+           n_bytes_read = svm_fifo_dequeue (ctx->s->tx_fifo,
+                                            len_to_deq, data);
        }
       ASSERT (n_bytes_read == len_to_deq);
       chain_b->current_length = n_bytes_read;
@@ -462,7 +467,7 @@ session_tx_fill_buffer (vlib_main_t * vm, session_tx_context_t * ctx,
     }
   else
     {
-      if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
+      if (ctx->transport_vft->transport_options.tx_type == TRANSPORT_TX_DGRAM)
        {
          session_dgram_hdr_t *hdr = &ctx->hdr;
          svm_fifo_t *f = ctx->s->tx_fifo;
@@ -490,8 +495,8 @@ session_tx_fill_buffer (vlib_main_t * vm, session_tx_context_t * ctx,
        }
       else
        {
-         n_bytes_read = svm_fifo_dequeue_nowait (ctx->s->tx_fifo,
-                                                 len_to_deq, data0);
+         n_bytes_read = svm_fifo_dequeue (ctx->s->tx_fifo,
+                                          len_to_deq, data0);
          ASSERT (n_bytes_read > 0);
        }
     }
@@ -503,15 +508,6 @@ session_tx_fill_buffer (vlib_main_t * vm, session_tx_context_t * ctx,
    */
   if (PREDICT_FALSE (ctx->n_bufs_per_seg > 1 && ctx->left_to_snd))
     session_tx_fifo_chain_tail (vm, ctx, b, n_bufs, peek_data);
-
-  /* *INDENT-OFF* */
-  SESSION_EVT_DBG(SESSION_EVT_DEQ, ctx->s, ({
-       ed->data[0] = FIFO_EVENT_APP_TX;
-       ed->data[1] = ctx->max_dequeue;
-       ed->data[2] = len_to_deq;
-       ed->data[3] = ctx->left_to_snd;
-  }));
-  /* *INDENT-ON* */
 }
 
 always_inline u8
@@ -554,7 +550,7 @@ session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx,
                               u32 max_segs, u8 peek_data)
 {
   u32 n_bytes_per_buf, n_bytes_per_seg;
-  ctx->max_dequeue = svm_fifo_max_dequeue (ctx->s->tx_fifo);
+  ctx->max_dequeue = svm_fifo_max_dequeue_cons (ctx->s->tx_fifo);
   if (peek_data)
     {
       /* Offset in rx fifo from where to peek data */
@@ -568,7 +564,7 @@ session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx,
     }
   else
     {
-      if (ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
+      if (ctx->transport_vft->transport_options.tx_type == TRANSPORT_TX_DGRAM)
        {
          if (ctx->max_dequeue <= sizeof (ctx->hdr))
            {
@@ -617,15 +613,17 @@ session_tx_set_dequeue_params (vlib_main_t * vm, session_tx_context_t * ctx,
 }
 
 always_inline int
-session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
-                               session_manager_worker_t * wrk,
-                               session_event_t * e, int *n_tx_packets,
-                               u8 peek_data)
+session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
+                               vlib_node_runtime_t * node,
+                               session_evt_elt_t * elt,
+                               int *n_tx_packets, u8 peek_data)
 {
-  u32 next_index, next0, next1, *to_next, n_left_to_next, n_left, pbi;
-  u32 n_trace = vlib_get_trace_count (vm, node), n_bufs_needed = 0;
-  session_manager_main_t *smm = &session_manager_main;
+  u32 next_index, next0, next1, *to_next, n_left_to_next, max_burst;
+  u32 n_trace, n_bufs_needed = 0, n_left, pbi;
   session_tx_context_t *ctx = &wrk->ctx;
+  session_main_t *smm = &session_main;
+  session_event_t *e = &elt->evt;
+  vlib_main_t *vm = wrk->vm;
   transport_proto_t tp;
   vlib_buffer_t *pb;
   u16 n_bufs, rv;
@@ -633,12 +631,13 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
   if (PREDICT_FALSE ((rv = session_tx_not_ready (ctx->s, peek_data))))
     {
       if (rv < 2)
-       vec_add1 (wrk->pending_event_vector, *e);
+       session_evt_add_old (wrk, elt);
       return SESSION_TX_NO_DATA;
     }
 
   next_index = smm->session_type_to_next[ctx->s->session_type];
   next0 = next1 = next_index;
+  max_burst = VLIB_FRAME_SIZE - *n_tx_packets;
 
   tp = session_get_transport_proto (ctx->s);
   ctx->transport_vft = transport_protocol_get_vft (tp);
@@ -651,13 +650,28 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
        ctx->transport_vft->flush_data (ctx->tc);
     }
 
+  if (ctx->s->flags & SESSION_F_CUSTOM_TX)
+    {
+      u32 n_custom_tx;
+      ctx->s->flags &= ~SESSION_F_CUSTOM_TX;
+      n_custom_tx = ctx->transport_vft->custom_tx (ctx->tc, max_burst);
+      *n_tx_packets += n_custom_tx;
+      max_burst -= n_custom_tx;
+      if (!max_burst)
+       {
+         session_evt_add_old (wrk, elt);
+         return SESSION_TX_OK;
+       }
+    }
+
   ctx->snd_space = transport_connection_snd_space (ctx->tc,
-                                                  vm->clib_time.
+                                                  wrk->vm->clib_time.
                                                   last_cpu_time,
                                                   ctx->snd_mss);
+
   if (ctx->snd_space == 0 || ctx->snd_mss == 0)
     {
-      vec_add1 (wrk->pending_event_vector, *e);
+      session_evt_add_old (wrk, elt);
       return SESSION_TX_NO_DATA;
     }
 
@@ -665,8 +679,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
   svm_fifo_unset_event (ctx->s->tx_fifo);
 
   /* Check how much we can pull. */
-  session_tx_set_dequeue_params (vm, ctx, VLIB_FRAME_SIZE - *n_tx_packets,
-                                peek_data);
+  session_tx_set_dequeue_params (vm, ctx, max_burst, peek_data);
 
   if (PREDICT_FALSE (!ctx->max_len_to_snd))
     return SESSION_TX_NO_DATA;
@@ -679,7 +692,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
     {
       if (n_bufs)
        vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
-      vec_add1 (wrk->pending_event_vector, *e);
+      session_evt_add_old (wrk, elt);
       return SESSION_TX_NO_BUFFERS;
     }
 
@@ -760,118 +773,204 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
                                       n_left_to_next, bi0, next0);
     }
 
-  if (PREDICT_FALSE (n_trace > 0))
+  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node)) > 0))
     session_tx_trace_frame (vm, node, next_index, to_next,
                            ctx->n_segs_per_evt, ctx->s, n_trace);
+
   if (PREDICT_FALSE (n_bufs))
-    {
-      clib_warning ("not all buffers consumed");
-      vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
-    }
+    vlib_buffer_free (vm, wrk->tx_buffers, n_bufs);
+
   *n_tx_packets += ctx->n_segs_per_evt;
   transport_connection_update_tx_stats (ctx->tc, ctx->max_len_to_snd);
   vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 
+  SESSION_EVT (SESSION_EVT_DEQ, ctx->s, ctx->max_len_to_snd, ctx->max_dequeue,
+              ctx->s->tx_fifo->has_event, wrk->last_vlib_time);
+
   /* If we couldn't dequeue all bytes mark as partially read */
   ASSERT (ctx->left_to_snd == 0);
   if (ctx->max_len_to_snd < ctx->max_dequeue)
     if (svm_fifo_set_event (ctx->s->tx_fifo))
-      vec_add1 (wrk->pending_event_vector, *e);
+      session_evt_add_old (wrk, elt);
 
-  if (!peek_data && ctx->transport_vft->tx_type == TRANSPORT_TX_DGRAM)
+  if (!peek_data
+      && ctx->transport_vft->transport_options.tx_type == TRANSPORT_TX_DGRAM)
     {
       /* Fix dgram pre header */
       if (ctx->max_len_to_snd < ctx->max_dequeue)
        svm_fifo_overwrite_head (ctx->s->tx_fifo, (u8 *) & ctx->hdr,
                                 sizeof (session_dgram_pre_hdr_t));
       /* More data needs to be read */
-      else if (svm_fifo_max_dequeue (ctx->s->tx_fifo) > 0)
+      else if (svm_fifo_max_dequeue_cons (ctx->s->tx_fifo) > 0)
        if (svm_fifo_set_event (ctx->s->tx_fifo))
-         vec_add1 (wrk->pending_event_vector, *e);
+         session_evt_add_old (wrk, elt);
+
+      if (svm_fifo_needs_deq_ntf (ctx->s->tx_fifo, ctx->max_len_to_snd))
+       session_dequeue_notify (ctx->s);
     }
   return SESSION_TX_OK;
 }
 
 int
-session_tx_fifo_peek_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node,
-                             session_manager_worker_t * wrk,
-                             session_event_t * e, int *n_tx_pkts)
+session_tx_fifo_peek_and_snd (session_worker_t * wrk,
+                             vlib_node_runtime_t * node,
+                             session_evt_elt_t * e, int *n_tx_packets)
 {
-  return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 1);
+  return session_tx_fifo_read_and_snd_i (wrk, node, e, n_tx_packets, 1);
 }
 
 int
-session_tx_fifo_dequeue_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node,
-                                session_manager_worker_t * wrk,
-                                session_event_t * e, int *n_tx_pkts)
+session_tx_fifo_dequeue_and_snd (session_worker_t * wrk,
+                                vlib_node_runtime_t * node,
+                                session_evt_elt_t * e, int *n_tx_packets)
 {
-  return session_tx_fifo_read_and_snd_i (vm, node, wrk, e, n_tx_pkts, 0);
+  return session_tx_fifo_read_and_snd_i (wrk, node, e, n_tx_packets, 0);
 }
 
 int
-session_tx_fifo_dequeue_internal (vlib_main_t * vm,
+session_tx_fifo_dequeue_internal (session_worker_t * wrk,
                                  vlib_node_runtime_t * node,
-                                 session_manager_worker_t * wrk,
-                                 session_event_t * e, int *n_tx_pkts)
+                                 session_evt_elt_t * e, int *n_tx_packets)
 {
   session_t *s = wrk->ctx.s;
-  application_t *app;
 
-  if (PREDICT_FALSE (s->session_state == SESSION_STATE_CLOSED))
+  if (PREDICT_FALSE (s->session_state >= SESSION_STATE_TRANSPORT_CLOSED))
     return 0;
-  app = application_get (s->t_app_index);
   svm_fifo_unset_event (s->tx_fifo);
-  return app->cb_fns.builtin_app_tx_callback (s);
+  return transport_custom_tx (session_get_transport_proto (s), s,
+                             VLIB_FRAME_SIZE - *n_tx_packets);
 }
 
 always_inline session_t *
 session_event_get_session (session_event_t * e, u8 thread_index)
 {
-  return session_get_if_valid (e->fifo->master_session_index, thread_index);
+  return session_get_if_valid (e->session_index, thread_index);
 }
 
-static void
-session_update_dispatch_period (session_manager_worker_t * wrk, f64 now,
-                               u32 thread_index)
+always_inline void
+session_event_dispatch (session_worker_t * wrk, vlib_node_runtime_t * node,
+                       session_evt_elt_t * elt, u32 thread_index,
+                       int *n_tx_packets)
 {
-  if (wrk->last_tx_packets)
+  session_main_t *smm = &session_main;
+  app_worker_t *app_wrk;
+  void (*fp) (void *);
+  session_event_t *e;
+  session_t *s;
+  int rv;
+
+  e = &elt->evt;
+  switch (e->event_type)
     {
-      f64 sample = now - wrk->last_vlib_time;
-      wrk->dispatch_period = (wrk->dispatch_period + sample) * 0.5;
+    case SESSION_IO_EVT_TX_FLUSH:
+    case SESSION_IO_EVT_TX:
+      /* Don't try to send more that one frame per dispatch cycle */
+      if (*n_tx_packets == VLIB_FRAME_SIZE)
+       {
+         session_evt_add_postponed (wrk, elt);
+         return;
+       }
+
+      s = session_event_get_session (e, thread_index);
+      if (PREDICT_FALSE (!s))
+       {
+         clib_warning ("session was freed!");
+         break;
+       }
+      CLIB_PREFETCH (s->tx_fifo, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+      wrk->ctx.s = s;
+      /* Spray packets in per session type frames, since they go to
+       * different nodes */
+      rv = (smm->session_tx_fns[s->session_type]) (wrk, node, elt,
+                                                  n_tx_packets);
+      if (PREDICT_FALSE (rv == SESSION_TX_NO_BUFFERS))
+       {
+         vlib_node_increment_counter (wrk->vm, node->node_index,
+                                      SESSION_QUEUE_ERROR_NO_BUFFER, 1);
+         break;
+       }
+      break;
+    case SESSION_IO_EVT_RX:
+      s = session_event_get_session (e, thread_index);
+      if (!s)
+       break;
+      transport_app_rx_evt (session_get_transport_proto (s),
+                           s->connection_index, s->thread_index);
+      break;
+    case SESSION_CTRL_EVT_CLOSE:
+      s = session_get_from_handle_if_valid (e->session_handle);
+      if (PREDICT_FALSE (!s))
+       break;
+      session_transport_close (s);
+      break;
+    case SESSION_IO_EVT_BUILTIN_RX:
+      s = session_event_get_session (e, thread_index);
+      if (PREDICT_FALSE (!s || s->session_state >= SESSION_STATE_CLOSING))
+       break;
+      svm_fifo_unset_event (s->rx_fifo);
+      app_wrk = app_worker_get (s->app_wrk_index);
+      app_worker_builtin_rx (app_wrk, s);
+      break;
+    case SESSION_IO_EVT_BUILTIN_TX:
+      s = session_get_from_handle_if_valid (e->session_handle);
+      wrk->ctx.s = s;
+      if (PREDICT_TRUE (s != 0))
+       session_tx_fifo_dequeue_internal (wrk, node, elt, n_tx_packets);
+      break;
+    case SESSION_CTRL_EVT_RPC:
+      fp = e->rpc_args.fp;
+      (*fp) (e->rpc_args.arg);
+      break;
+    case SESSION_CTRL_EVT_DISCONNECTED:
+      session_mq_disconnected_handler (e->data);
+      break;
+    case SESSION_CTRL_EVT_ACCEPTED_REPLY:
+      session_mq_accepted_reply_handler (e->data);
+      break;
+    case SESSION_CTRL_EVT_CONNECTED_REPLY:
+      break;
+    case SESSION_CTRL_EVT_DISCONNECTED_REPLY:
+      session_mq_disconnected_reply_handler (e->data);
+      break;
+    case SESSION_CTRL_EVT_RESET_REPLY:
+      session_mq_reset_reply_handler (e->data);
+      break;
+    case SESSION_CTRL_EVT_WORKER_UPDATE:
+      session_mq_worker_update_handler (e->data);
+      break;
+    default:
+      clib_warning ("unhandled event type %d", e->event_type);
     }
-  wrk->last_vlib_time = now;
 }
 
 static uword
 session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
                       vlib_frame_t * frame)
 {
-  session_manager_main_t *smm = vnet_get_session_manager_main ();
-  u32 thread_index = vm->thread_index, n_to_dequeue, n_events;
-  session_manager_worker_t *wrk = &smm->wrk[thread_index];
-  session_event_t *e, *fifo_events;
+  session_main_t *smm = vnet_get_session_main ();
+  u32 thread_index = vm->thread_index, n_to_dequeue;
+  session_worker_t *wrk = &smm->wrk[thread_index];
+  session_evt_elt_t *elt, *new_he, *new_te, *old_he;
+  session_evt_elt_t *disconnects_he, *postponed_he;
   svm_msg_q_msg_t _msg, *msg = &_msg;
-  f64 now = vlib_time_now (vm);
-  int n_tx_packets = 0, i, rv;
-  app_worker_t *app_wrk;
-  application_t *app;
+  int i, n_tx_packets = 0;
   svm_msg_q_t *mq;
-  void (*fp) (void *);
 
-  SESSION_EVT_DBG (SESSION_EVT_POLL_GAP_TRACK, smm, thread_index);
+  SESSION_EVT (SESSION_EVT_DISPATCH_START, wrk);
+
+  wrk->last_vlib_time = vlib_time_now (vm);
 
   /*
    *  Update transport time
    */
-  session_update_dispatch_period (wrk, now, thread_index);
-  transport_update_time (now, thread_index);
-
-  SESSION_EVT_DBG (SESSION_EVT_DEQ_NODE, 0);
+  transport_update_time (wrk->last_vlib_time, thread_index);
 
   /* Make sure postponed events are handled first */
-  fifo_events = wrk->free_event_vector;
-  vec_append (fifo_events, wrk->postponed_event_vector);
-  _vec_len (wrk->postponed_event_vector) = 0;
+  new_he = pool_elt_at_index (wrk->event_elts, wrk->new_head);
+  new_te = clib_llist_prev (wrk->event_elts, evt_list, new_he);
+
+  postponed_he = pool_elt_at_index (wrk->event_elts, wrk->postponed_head);
+  clib_llist_splice (wrk->event_elts, evt_list, new_te, postponed_he);
 
   /* Try to dequeue what is available. Don't wait for lock.
    * XXX: we may need priorities here */
@@ -881,139 +980,49 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
     {
       for (i = 0; i < n_to_dequeue; i++)
        {
-         vec_add2 (fifo_events, e, 1);
+         elt = session_evt_elt_alloc (wrk);
          svm_msg_q_sub_w_lock (mq, msg);
          /* Works because reply messages are smaller than a session evt.
           * If we ever need to support bigger messages this needs to be
           * fixed */
-         clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), sizeof (*e));
+         clib_memcpy_fast (&elt->evt, svm_msg_q_msg_data (mq, msg),
+                           sizeof (elt->evt));
          svm_msg_q_free_msg (mq, msg);
+         new_he = pool_elt_at_index (wrk->event_elts, wrk->new_head);
+         clib_llist_add_tail (wrk->event_elts, evt_list, elt, new_he);
        }
       svm_msg_q_unlock (mq);
     }
 
-  vec_append (fifo_events, wrk->pending_event_vector);
-  vec_append (fifo_events, wrk->pending_disconnects);
+  old_he = pool_elt_at_index (wrk->event_elts, wrk->old_head);
+  disconnects_he = pool_elt_at_index (wrk->event_elts, wrk->disconnects_head);
 
-  _vec_len (wrk->pending_event_vector) = 0;
-  _vec_len (wrk->pending_disconnects) = 0;
+  new_te = clib_llist_prev (wrk->event_elts, evt_list, new_he);
+  clib_llist_splice (wrk->event_elts, evt_list, new_te, old_he);
+  new_te = clib_llist_prev (wrk->event_elts, evt_list, new_he);
+  clib_llist_splice (wrk->event_elts, evt_list, new_te, disconnects_he);
 
-  n_events = vec_len (fifo_events);
-  if (PREDICT_FALSE (!n_events))
-    return 0;
-
-  for (i = 0; i < n_events; i++)
+  while (!clib_llist_is_empty (wrk->event_elts, evt_list, new_he))
     {
-      session_t *s;            /* $$$ prefetch 1 ahead maybe */
-      session_event_t *e;
-      u8 need_tx_ntf;
+      clib_llist_index_t ei;
 
-      e = &fifo_events[i];
-      switch (e->event_type)
-       {
-       case SESSION_IO_EVT_TX_FLUSH:
-       case FIFO_EVENT_APP_TX:
-         /* Don't try to send more that one frame per dispatch cycle */
-         if (n_tx_packets == VLIB_FRAME_SIZE)
-           {
-             vec_add1 (wrk->postponed_event_vector, *e);
-             break;
-           }
+      clib_llist_pop_first (wrk->event_elts, evt_list, elt, new_he);
+      ei = clib_llist_entry_index (wrk->event_elts, elt);
 
-         s = session_event_get_session (e, thread_index);
-         if (PREDICT_FALSE (!s))
-           {
-             clib_warning ("session was freed!");
-             continue;
-           }
-         wrk->ctx.s = s;
-         /* Spray packets in per session type frames, since they go to
-          * different nodes */
-         rv = (smm->session_tx_fns[s->session_type]) (vm, node, wrk, e,
-                                                      &n_tx_packets);
-         if (PREDICT_TRUE (rv == SESSION_TX_OK))
-           {
-             need_tx_ntf = svm_fifo_needs_tx_ntf (s->tx_fifo,
-                                                  wrk->ctx.max_len_to_snd);
-             if (PREDICT_FALSE (need_tx_ntf))
-               session_dequeue_notify (s);
-           }
-         else if (PREDICT_FALSE (rv == SESSION_TX_NO_BUFFERS))
-           {
-             vlib_node_increment_counter (vm, node->node_index,
-                                          SESSION_QUEUE_ERROR_NO_BUFFER, 1);
-             continue;
-           }
-         break;
-       case FIFO_EVENT_DISCONNECT:
-         s = session_get_from_handle_if_valid (e->session_handle);
-         if (PREDICT_FALSE (!s))
-           break;
-
-         /* Make sure session disconnects run after the pending list is
-          * drained, i.e., postpone if the first time. If not the first
-          * and the tx queue is still not empty, try to wait for some
-          * dispatch cycles */
-         if (!e->postponed
-             || (e->postponed < 200 && svm_fifo_max_dequeue (s->tx_fifo)))
-           {
-             e->postponed += 1;
-             vec_add1 (wrk->pending_disconnects, *e);
-             continue;
-           }
+      session_event_dispatch (wrk, node, elt, thread_index, &n_tx_packets);
 
-         session_transport_close (s);
-         break;
-       case FIFO_EVENT_BUILTIN_RX:
-         s = session_event_get_session (e, thread_index);
-         if (PREDICT_FALSE (!s || s->session_state >= SESSION_STATE_CLOSING))
-           continue;
-         svm_fifo_unset_event (s->rx_fifo);
-         app_wrk = app_worker_get (s->app_wrk_index);
-         app = application_get (app_wrk->app_index);
-         app->cb_fns.builtin_app_rx_callback (s);
-         break;
-       case FIFO_EVENT_BUILTIN_TX:
-         s = session_get_from_handle_if_valid (e->session_handle);
-         wrk->ctx.s = s;
-         if (PREDICT_TRUE (s != 0))
-           session_tx_fifo_dequeue_internal (vm, node, wrk, e,
-                                             &n_tx_packets);
-         break;
-       case FIFO_EVENT_RPC:
-         fp = e->rpc_args.fp;
-         (*fp) (e->rpc_args.arg);
-         break;
-       case SESSION_CTRL_EVT_DISCONNECTED:
-         session_mq_disconnected_handler (e->data);
-         break;
-       case SESSION_CTRL_EVT_ACCEPTED_REPLY:
-         session_mq_accepted_reply_handler (e->data);
-         break;
-       case SESSION_CTRL_EVT_CONNECTED_REPLY:
-         break;
-       case SESSION_CTRL_EVT_DISCONNECTED_REPLY:
-         session_mq_disconnected_reply_handler (e->data);
-         break;
-       case SESSION_CTRL_EVT_RESET_REPLY:
-         session_mq_reset_reply_handler (e->data);
-         break;
-       case SESSION_CTRL_EVT_WORKER_UPDATE:
-         session_mq_worker_update_handler (e->data);
-         break;
-       default:
-         clib_warning ("unhandled event type %d", e->event_type);
-       }
-    }
+      /* Regrab elements in case pool moved */
+      elt = pool_elt_at_index (wrk->event_elts, ei);
+      if (!clib_llist_elt_is_linked (elt, evt_list))
+       session_evt_elt_free (wrk, elt);
 
-  _vec_len (fifo_events) = 0;
-  wrk->free_event_vector = fifo_events;
-  wrk->last_tx_packets = n_tx_packets;
+      new_he = pool_elt_at_index (wrk->event_elts, wrk->new_head);
+    }
 
   vlib_node_increment_counter (vm, session_queue_node.index,
                               SESSION_QUEUE_ERROR_TX, n_tx_packets);
 
-  SESSION_EVT_DBG (SESSION_EVT_DISPATCH_END, smm, thread_index);
+  SESSION_EVT (SESSION_EVT_DISPATCH_END, wrk, n_tx_packets);
 
   return n_tx_packets;
 }
@@ -1034,7 +1043,7 @@ VLIB_REGISTER_NODE (session_queue_node) =
 void
 dump_thread_0_event_queue (void)
 {
-  session_manager_main_t *smm = vnet_get_session_manager_main ();
+  session_main_t *smm = vnet_get_session_main ();
   vlib_main_t *vm = &vlib_global_main;
   u32 my_thread_index = vm->thread_index;
   session_event_t _e, *e = &_e;
@@ -1055,23 +1064,23 @@ dump_thread_0_event_queue (void)
 
       switch (e->event_type)
        {
-       case FIFO_EVENT_APP_TX:
+       case SESSION_IO_EVT_TX:
          s0 = session_event_get_session (e, my_thread_index);
          fformat (stdout, "[%04d] TX session %d\n", i, s0->session_index);
          break;
 
-       case FIFO_EVENT_DISCONNECT:
+       case SESSION_CTRL_EVT_CLOSE:
          s0 = session_get_from_handle (e->session_handle);
          fformat (stdout, "[%04d] disconnect session %d\n", i,
                   s0->session_index);
          break;
 
-       case FIFO_EVENT_BUILTIN_RX:
+       case SESSION_IO_EVT_BUILTIN_RX:
          s0 = session_event_get_session (e, my_thread_index);
          fformat (stdout, "[%04d] builtin_rx %d\n", i, s0->session_index);
          break;
 
-       case FIFO_EVENT_RPC:
+       case SESSION_CTRL_EVT_RPC:
          fformat (stdout, "[%04d] RPC call %llx with %llx\n",
                   i, (u64) (uword) (e->rpc_args.fp),
                   (u64) (uword) (e->rpc_args.arg));
@@ -1096,15 +1105,15 @@ session_node_cmp_event (session_event_t * e, svm_fifo_t * f)
   session_t *s;
   switch (e->event_type)
     {
-    case FIFO_EVENT_APP_RX:
-    case FIFO_EVENT_APP_TX:
-    case FIFO_EVENT_BUILTIN_RX:
-      if (e->fifo == f)
+    case SESSION_IO_EVT_RX:
+    case SESSION_IO_EVT_TX:
+    case SESSION_IO_EVT_BUILTIN_RX:
+      if (e->session_index == f->master_session_index)
        return 1;
       break;
-    case FIFO_EVENT_DISCONNECT:
+    case SESSION_CTRL_EVT_CLOSE:
       break;
-    case FIFO_EVENT_RPC:
+    case SESSION_CTRL_EVT_RPC:
       s = session_get_from_handle (e->session_handle);
       if (!s)
        {
@@ -1123,8 +1132,8 @@ session_node_cmp_event (session_event_t * e, svm_fifo_t * f)
 u8
 session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e)
 {
-  session_event_t *pending_event_vector, *evt;
-  session_manager_worker_t *wrk;
+  session_evt_elt_t *elt;
+  session_worker_t *wrk;
   int i, index, found = 0;
   svm_msg_q_msg_t *msg;
   svm_msg_q_ring_t *ring;
@@ -1133,7 +1142,7 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e)
 
   ASSERT (e);
   thread_index = f->master_thread_index;
-  wrk = session_manager_get_worker (thread_index);
+  wrk = session_main_get_worker (thread_index);
 
   /*
    * Search evt queue
@@ -1154,16 +1163,20 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e)
   /*
    * Search pending events vector
    */
-  pending_event_vector = wrk->pending_event_vector;
-  vec_foreach (evt, pending_event_vector)
-  {
-    found = session_node_cmp_event (evt, f);
+
+  /* *INDENT-OFF* */
+  clib_llist_foreach (wrk->event_elts, evt_list,
+                      session_evt_old_head (wrk), elt, ({
+    found = session_node_cmp_event (&elt->evt, f);
     if (found)
       {
-       clib_memcpy_fast (e, evt, sizeof (*evt));
+       clib_memcpy_fast (e, &elt->evt, sizeof (*e));
        break;
       }
-  }
+
+  }));
+  /* *INDENT-ON* */
+
   return found;
 }
 
@@ -1230,6 +1243,26 @@ VLIB_REGISTER_NODE (session_queue_process_node) =
 };
 /* *INDENT-ON* */
 
+static_always_inline uword
+session_queue_pre_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+                               vlib_frame_t * frame)
+{
+  session_main_t *sm = &session_main;
+  if (!sm->wrk[0].vpp_event_queue)
+    return 0;
+  return session_queue_node_fn (vm, node, frame);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (session_queue_pre_input_node) =
+{
+  .function = session_queue_pre_input_inline,
+  .type = VLIB_NODE_TYPE_PRE_INPUT,
+  .name = "session-queue-main",
+  .state = VLIB_NODE_STATE_DISABLED,
+};
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *