X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvcl%2Fvppcom.c;h=05b84d4674bdabb571f845ae9f090bc73a20fb4e;hb=dd92bdeb0735dac940d32f651c0b0b9ed65a6d7d;hp=ce4513c80141a3ab4d660813a5b19780f3921f9e;hpb=8d42c75d6c88be62ac7eed03f10305bba3423f7c;p=vpp.git

diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index ce4513c8014..05b84d4674b 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -289,10 +289,17 @@ vcl_session_transport_attr (vcl_worker_t *wrk, vcl_session_t *s, u8 is_get,
   f64 timeout;
 
   ASSERT (!wrk->session_attr_op);
+  mq = s->vpp_evt_q;
+  if (PREDICT_FALSE (!mq))
+    {
+      /* FIXME: attribute should be stored and sent once session is
+       * bound/connected to vpp */
+      return 0;
+    }
+
   wrk->session_attr_op = 1;
   wrk->session_attr_op_rv = -1;
 
-  mq = s->vpp_evt_q;
   app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_TRANSPORT_ATTR);
   mp = (session_transport_attr_msg_t *) app_evt->evt->data;
   memset (mp, 0, sizeof (*mp));
@@ -336,7 +343,8 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
 	mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
 	mp->vpp_event_queue_address, mp->mq_index, 0, session))
     {
-      VDBG (0, "failed to attach fifos for %u", session->session_index);
+      VDBG (0, "session %u [0x%llx]: failed to attach fifos",
+	    session->session_index, mp->handle);
       goto error;
     }
 
@@ -355,13 +363,6 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
   session->listener_index = listen_session->session_index;
   listen_session->n_accepted_sessions++;
 
-  VDBG (1,
-	"session %u [0x%llx]: client accept request from %s address %U"
-	" port %d queue %p!",
-	session->session_index, mp->handle, mp->rmt.is_ip4 ? "IPv4" : "IPv6",
-	vcl_format_ip46_address, &mp->rmt.ip,
-	mp->rmt.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
-	clib_net_to_host_u16 (mp->rmt.port), session->vpp_evt_q);
   vcl_evt (VCL_EVT_ACCEPT, session, listen_session, session_index);
 
   vcl_send_session_accepted_reply (session->vpp_evt_q, mp->context,
@@ -406,10 +407,11 @@ vcl_session_connected_handler (vcl_worker_t * wrk,
 
   if (mp->retval)
     {
-      VDBG (0, "session index %u: connect failed! %U", session_index,
+      VDBG (0, "session %u: connect failed! %U", session_index,
 	    format_session_error, mp->retval);
       session->session_state = VCL_STATE_DETACHED;
       session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
+      session->vpp_error = mp->retval;
       return session_index;
     }
 
@@ -424,7 +426,8 @@ vcl_session_connected_handler (vcl_worker_t * wrk,
 	mp->segment_handle, mp->server_rx_fifo, mp->server_tx_fifo,
 	mp->vpp_event_queue_address, mp->mq_index, 0, session))
     {
-      VDBG (0, "failed to attach fifos for %u", session->session_index);
+      VDBG (0, "session %u [0x%llx]: failed to attach fifos",
+	    session->session_index, session->vpp_handle);
       session->session_state = VCL_STATE_UPDATED;
       vcl_send_session_disconnect (wrk, session);
       return session_index;
@@ -436,7 +439,8 @@ vcl_session_connected_handler (vcl_worker_t * wrk,
 				      mp->ct_tx_fifo, (uword) ~0, ~0, 1,
 				      session))
 	{
-	  VDBG (0, "failed to attach ct fifos for %u", session->session_index);
+	  VDBG (0, "session %u [0x%llx]: failed to attach ct fifos",
+		session->session_index, session->vpp_handle);
 	  session->session_state = VCL_STATE_UPDATED;
 	  vcl_send_session_disconnect (wrk, session);
 	  return session_index;
@@ -455,9 +459,14 @@ vcl_session_connected_handler (vcl_worker_t * wrk,
   else
     session->session_state = VCL_STATE_READY;
 
-  VDBG (1, "session %u [0x%llx] connected! rx_fifo %p, refcnt %d, tx_fifo %p,"
-	" refcnt %d", session_index, mp->handle, session->rx_fifo,
-	session->rx_fifo->refcnt, session->tx_fifo, session->tx_fifo->refcnt);
+  VDBG (0, "session %u [0x%llx] connected local: %U:%u remote %U:%u",
+	session->session_index, session->vpp_handle, vcl_format_ip46_address,
+	&session->transport.lcl_ip,
+	session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
+	clib_net_to_host_u16 (session->transport.lcl_port),
+	vcl_format_ip46_address, &session->transport.rmt_ip,
+	session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
+	clib_net_to_host_u16 (session->transport.rmt_port));
 
   return session_index;
 }
@@ -507,7 +516,7 @@ vcl_session_reset_handler (vcl_worker_t * wrk,
 
   if (session->session_state != VCL_STATE_CLOSED)
     session->session_state = VCL_STATE_DISCONNECT;
-  VDBG (0, "reset session %u [0x%llx]", sid, reset_msg->handle);
+  VDBG (0, "session %u [0x%llx]: reset", sid, reset_msg->handle);
   return sid;
 }
 
@@ -530,8 +539,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
 	}
       else
 	{
-	  VDBG (0, "ERROR: session %u [0x%llx]: Invalid session index!",
-		sid, mp->handle);
+	  VDBG (0, "session %u [0x%llx]: Invalid session index!", sid,
+		mp->handle);
 	  return VCL_INVALID_SESSION_INDEX;
 	}
     }
@@ -550,7 +559,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
 				      mp->tx_fifo, mp->vpp_evt_q, mp->mq_index,
 				      0, session))
 	{
-	  VDBG (0, "failed to attach fifos for %u", session->session_index);
+	  VDBG (0, "session %u [0x%llx]: failed to attach fifos",
+		session->session_index, session->vpp_handle);
 	  session->session_state = VCL_STATE_DETACHED;
 	  return VCL_INVALID_SESSION_INDEX;
 	}
@@ -676,7 +686,7 @@ vcl_session_disconnected_handler (vcl_worker_t * wrk,
   session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
   if (!session)
     {
-      VDBG (0, "request to disconnect unknown handle 0x%llx", msg->handle);
+      VWRN ("request to disconnect unknown handle 0x%llx", msg->handle);
       return 0;
     }
 
@@ -758,8 +768,8 @@ vppcom_session_disconnect (u32 session_handle)
   vpp_handle = session->vpp_handle;
   state = session->session_state;
 
-  VDBG (1, "session %u [0x%llx] state 0x%x (%s)", session->session_index,
-	vpp_handle, state, vcl_session_state_str (state));
+  VDBG (1, "session %u [0x%llx]: disconnecting state (%s)",
+	session->session_index, vpp_handle, vcl_session_state_str (state));
 
   if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
     {
@@ -780,7 +790,7 @@ vppcom_session_disconnect (u32 session_handle)
       if (PREDICT_FALSE (!session->vpp_evt_q))
 	return VPPCOM_OK;
 
-      VDBG (1, "session %u [0x%llx]: sending disconnect...",
+      VDBG (1, "session %u [0x%llx]: sending disconnect",
 	    session->session_index, vpp_handle);
       vcl_send_session_disconnect (wrk, session);
     }
@@ -788,7 +798,8 @@ vppcom_session_disconnect (u32 session_handle)
   if (session->listener_index != VCL_INVALID_SESSION_INDEX)
     {
       listen_session = vcl_session_get (wrk, session->listener_index);
-      listen_session->n_accepted_sessions--;
+      if (listen_session)
+	listen_session->n_accepted_sessions--;
     }
 
   return VPPCOM_OK;
@@ -804,7 +815,7 @@ vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
   session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
   if (!session)
     {
-      VDBG (0, "disconnect confirmed for unknown handle 0x%llx", msg->handle);
+      VWRN ("disconnect confirmed for unknown handle 0x%llx", msg->handle);
       return;
     }
 
@@ -835,15 +846,28 @@ vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
       return;
     }
 
+  /* VPP will reuse the handle so clean it up now */
   vcl_session_table_del_vpp_handle (wrk, msg->handle);
-  /* Should not happen. App did not close the connection so don't free it. */
+
+  /* App did not close the connection yet so don't free it. */
   if (session->session_state != VCL_STATE_CLOSED)
     {
-      VDBG (0, "app did not close session %d", session->session_index);
+      VDBG (0, "session %u: app did not close", session->session_index);
       session->session_state = VCL_STATE_DETACHED;
       session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
       return;
     }
+
+  /* Session probably tracked with epoll, disconnect not yet handled and
+   * 1) both transport and session cleanup completed 2) app closed. Wait
+   * until message is drained to free the session.
+   * See @ref vcl_handle_mq_event */
+  if (session->flags & VCL_SESSION_F_PENDING_DISCONNECT)
+    {
+      session->flags |= VCL_SESSION_F_PENDING_FREE;
+      return;
+    }
+
   vcl_session_free (wrk, session);
 }
 
@@ -1016,9 +1040,16 @@ vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
       disconnected_msg = (session_disconnected_msg_t *) e->data;
       if (!(s = vcl_session_get_w_vpp_handle (wrk, disconnected_msg->handle)))
 	break;
+      if (s->session_state == VCL_STATE_CLOSED)
+	break;
       if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
 	{
-	  vec_add1 (wrk->unhandled_evts_vector, *e);
+	  s->session_state = VCL_STATE_VPP_CLOSING;
+	  s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
+	  vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
+	  *ecpy = *e;
+	  ecpy->postponed = 1;
+	  ecpy->session_index = s->session_index;
 	  break;
 	}
       if (!(s = vcl_session_disconnected_handler (wrk, disconnected_msg)))
@@ -1030,9 +1061,16 @@ vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
       reset_msg = (session_reset_msg_t *) e->data;
       if (!(s = vcl_session_get_w_vpp_handle (wrk, reset_msg->handle)))
 	break;
+      if (s->session_state == VCL_STATE_CLOSED)
+	break;
       if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
 	{
-	  vec_add1 (wrk->unhandled_evts_vector, *e);
+	  s->flags |= VCL_SESSION_F_PENDING_DISCONNECT;
+	  s->session_state = VCL_STATE_DISCONNECT;
+	  vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
+	  *ecpy = *e;
+	  ecpy->postponed = 1;
+	  ecpy->session_index = s->session_index;
 	  break;
 	}
       vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
@@ -1227,9 +1265,39 @@ vcl_api_attach (void)
   return vcl_bapi_attach ();
 }
 
+static void
+vcl_api_retry_attach (vcl_worker_t *wrk)
+{
+  vcl_session_t *s;
+
+  if (vcl_api_attach ())
+    return;
+
+  /* Treat listeners as configuration that needs to be re-added to vpp */
+  pool_foreach (s, wrk->sessions)
+    {
+      if (s->flags & VCL_SESSION_F_IS_VEP)
+	continue;
+      if (s->session_state == VCL_STATE_LISTEN_NO_MQ)
+	vppcom_session_listen (vcl_session_handle (s), 10);
+      else
+	VDBG (0, "internal error: unexpected state %d", s->session_state);
+    }
+}
+
+static void
+vcl_api_handle_disconnect (vcl_worker_t *wrk)
+{
+  wrk->api_client_handle = ~0;
+  vcl_worker_detach_sessions (wrk);
+}
+
 static void
 vcl_api_detach (vcl_worker_t * wrk)
 {
+  if (wrk->api_client_handle == ~0)
+    return;
+
   vcl_send_app_detach (wrk);
 
   if (vcm->cfg.vpp_app_socket_api)
@@ -1272,7 +1340,10 @@ vppcom_app_create (const char *app_name)
   vcl_worker_alloc_and_init ();
 
   if ((rv = vcl_api_attach ()))
-    return rv;
+    {
+      vppcom_app_destroy ();
+      return rv;
+    }
 
   VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
 	vcm->workers[0].api_client_handle, vcm->workers[0].api_client_handle);
@@ -1302,6 +1373,7 @@ vppcom_app_destroy (void)
 
   vcl_api_detach (current_wrk);
   vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
+  vcl_set_worker_index (~0);
 
   vcl_elog_stop (vcm);
 
@@ -1327,6 +1399,7 @@ vppcom_session_create (u8 proto, u8 is_nonblocking)
   session->session_state = VCL_STATE_CLOSED;
   session->vpp_handle = ~0;
   session->is_dgram = vcl_proto_is_dgram (proto);
+  session->vpp_error = SESSION_E_NONE;
 
   if (is_nonblocking)
     vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
@@ -1449,9 +1522,14 @@ vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
     }
   else if (s->session_state == VCL_STATE_DETACHED)
     {
-      /* Should not happen. VPP cleaned up before app confirmed close */
       VDBG (0, "vpp freed session %d before close", s->session_index);
-      goto free_session;
+
+      if (!(s->flags & VCL_SESSION_F_PENDING_DISCONNECT))
+	goto free_session;
+
+      /* Disconnect/reset messages pending but vpp transport and session
+       * cleanups already done. Free only after messages drained. */
+      s->flags |= VCL_SESSION_F_PENDING_FREE;
     }
 
   s->session_state = VCL_STATE_CLOSED;
@@ -1740,22 +1818,20 @@ vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
 
   if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
     {
-      VDBG (0, "ERROR: cannot connect epoll session %u!",
-	    session->session_index);
+      VWRN ("cannot connect epoll session %u!", session->session_index);
       return VPPCOM_EBADFD;
     }
 
   if (PREDICT_FALSE (vcl_session_is_ready (session)))
     {
       VDBG (0,
-	    "session handle %u [0x%llx]: session already "
-	    "connected to %s %U port %d proto %s, state 0x%x (%s)",
-	    session_handle, session->vpp_handle,
-	    session->transport.is_ip4 ? "IPv4" : "IPv6",
+	    "session %u [0x%llx]: already connected to %U:%d proto %s,"
+	    " state (%s)",
+	    session->session_index, session->vpp_handle,
 	    vcl_format_ip46_address, &session->transport.rmt_ip,
 	    session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
 	    clib_net_to_host_u16 (session->transport.rmt_port),
-	    vppcom_proto_str (session->session_type), session->session_state,
+	    vppcom_proto_str (session->session_type),
 	    vcl_session_state_str (session->session_state));
       return VPPCOM_OK;
     }
@@ -1775,11 +1851,8 @@ vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
   session->parent_handle = VCL_INVALID_SESSION_HANDLE;
   session->flags |= VCL_SESSION_F_CONNECTED;
 
-  VDBG (0,
-	"session handle %u (%s): connecting to peer %s %U "
-	"port %d proto %s",
-	session_handle, vcl_session_state_str (session->session_state),
-	session->transport.is_ip4 ? "IPv4" : "IPv6", vcl_format_ip46_address,
+  VDBG (0, "session %u: connecting to peer %U:%d proto %s",
+	session->session_index, vcl_format_ip46_address,
 	&session->transport.rmt_ip,
 	session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
 	clib_net_to_host_u16 (session->transport.rmt_port),
@@ -2055,7 +2128,7 @@ vppcom_session_read_segments (uint32_t session_handle,
     }
 
   n_read = svm_fifo_segments (rx_fifo, s->rx_bytes_pending,
-			      (svm_fifo_seg_t *) ds, n_segments, max_bytes);
+			      (svm_fifo_seg_t *) ds, &n_segments, max_bytes);
   if (n_read < 0)
     return VPPCOM_EAGAIN;
 
@@ -2439,11 +2512,23 @@ vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits,
   int n_mq_evts, i;
   u64 buf;
 
+  if (PREDICT_FALSE (wrk->api_client_handle == ~0))
+    {
+      vcl_api_retry_attach (wrk);
+      return 0;
+    }
+
   vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
   n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
 			  vec_len (wrk->mq_events), time_to_wait);
   for (i = 0; i < n_mq_evts; i++)
     {
+      if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
+	{
+	  vcl_api_handle_disconnect (wrk);
+	  continue;
+	}
+
       mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
       n_read = read (mqc->mq_fd, &buf, sizeof (buf));
       vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map,
@@ -2925,7 +3010,7 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
     case SESSION_IO_EVT_TX:
       sid = e->session_index;
       s = vcl_session_get (wrk, sid);
-      if (vcl_session_is_closed (s))
+      if (!s || !vcl_session_is_open (s))
 	break;
       session_events = s->vep.ev.events;
       if (!(EPOLLOUT & session_events))
@@ -2982,10 +3067,15 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
       else
 	{
 	  s = vcl_session_get (wrk, e->session_index);
+	  s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
 	}
       if (vcl_session_is_closed (s) ||
 	  !(s->flags & VCL_SESSION_F_IS_VEP_SESSION))
-	break;
+	{
+	  if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
+	    vcl_session_free (wrk, s);
+	  break;
+	}
       sid = s->session_index;
       session_events = s->vep.ev.events;
       add_event = 1;
@@ -3008,13 +3098,24 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
       break;
     case SESSION_CTRL_EVT_RESET:
       if (!e->postponed)
-	sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
+	{
+	  sid =
+	    vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
+	  s = vcl_session_get (wrk, sid);
+	}
       else
-	sid = e->session_index;
-      s = vcl_session_get (wrk, sid);
+	{
+	  sid = e->session_index;
+	  s = vcl_session_get (wrk, sid);
+	  s->flags &= ~VCL_SESSION_F_PENDING_DISCONNECT;
+	}
       if (vcl_session_is_closed (s) ||
 	  !(s->flags & VCL_SESSION_F_IS_VEP_SESSION))
-	break;
+	{
+	  if (s && (s->flags & VCL_SESSION_F_PENDING_FREE))
+	    vcl_session_free (wrk, s);
+	  break;
+	}
       session_events = s->vep.ev.events;
       add_event = 1;
       events[*num_ev].events = EPOLLHUP | EPOLLRDHUP;
@@ -3144,6 +3245,12 @@ vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events,
   double end = -1;
   u64 buf;
 
+  if (PREDICT_FALSE (wrk->api_client_handle == ~0))
+    {
+      vcl_api_retry_attach (wrk);
+      return n_evts;
+    }
+
   vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
   if (!n_evts)
     {
@@ -3163,6 +3270,13 @@ vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events,
 
       for (i = 0; i < n_mq_evts; i++)
 	{
+	  if (PREDICT_FALSE (wrk->mq_events[i].data.u32 == ~0))
+	    {
+	      /* api socket was closed */
+	      vcl_api_handle_disconnect (wrk);
+	      continue;
+	    }
+
 	  mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
 	  n_read = read (mqc->mq_fd, &buf, sizeof (buf));
 	  vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0,
@@ -4308,6 +4422,10 @@ vppcom_retval_str (int retval)
       st = "VPPCOM_ETIMEDOUT";
       break;
 
+    case VPPCOM_EADDRINUSE:
+      st = "VPPCOM_EADDRINUSE";
+      break;
+
     default:
       st = "UNKNOWN_STATE";
       break;
@@ -4334,6 +4452,32 @@ vppcom_del_cert_key_pair (uint32_t ckpair_index)
     return vcl_bapi_del_cert_key_pair (ckpair_index);
 }
 
+int
+vppcom_session_get_error (uint32_t session_handle)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  vcl_session_t *session = 0;
+
+  session = vcl_session_get_w_handle (wrk, session_handle);
+  if (!session)
+    return VPPCOM_EBADFD;
+
+  if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
+    {
+      VWRN ("epoll session %u! will not have connect", session->session_index);
+      return VPPCOM_EBADFD;
+    }
+
+  if (session->vpp_error == SESSION_E_PORTINUSE)
+    return VPPCOM_EADDRINUSE;
+  else if (session->vpp_error == SESSION_E_REFUSED)
+    return VPPCOM_ECONNREFUSED;
+  else if (session->vpp_error != SESSION_E_NONE)
+    return VPPCOM_EFAULT;
+  else
+    return VPPCOM_OK;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *