2 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 #include <vcl/vppcom.h>
19 #include <vcl/vcl_debug.h>
20 #include <vcl/vcl_private.h>
21 #include <svm/fifo_segment.h>
23 __thread uword __vcl_worker_index = ~0;
26 vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg)
32 n_msgs = clib_min (svm_msg_q_size (mq), n_max_msg);
33 for (i = 0; i < n_msgs; i++)
35 vec_add2 (wrk->mq_msg_vector, msg, 1);
36 svm_msg_q_sub_w_lock (mq, msg);
42 vppcom_session_state_str (vcl_session_state_t state)
48 case VCL_STATE_CLOSED:
51 case VCL_STATE_LISTEN:
57 case VCL_STATE_VPP_CLOSING:
58 st = "STATE_VPP_CLOSING";
60 case VCL_STATE_DISCONNECT:
61 st = "STATE_DISCONNECT";
63 case VCL_STATE_DETACHED:
64 st = "STATE_DETACHED";
66 case VCL_STATE_UPDATED:
69 case VCL_STATE_LISTEN_NO_MQ:
70 st = "STATE_LISTEN_NO_MQ";
81 format_ip4_address (u8 * s, va_list * args)
83 u8 *a = va_arg (*args, u8 *);
84 return format (s, "%d.%d.%d.%d", a[0], a[1], a[2], a[3]);
88 format_ip6_address (u8 * s, va_list * args)
90 ip6_address_t *a = va_arg (*args, ip6_address_t *);
91 u32 i, i_max_n_zero, max_n_zeros, i_first_zero, n_zeros, last_double_colon;
93 i_max_n_zero = ARRAY_LEN (a->as_u16);
95 i_first_zero = i_max_n_zero;
97 for (i = 0; i < ARRAY_LEN (a->as_u16); i++)
99 u32 is_zero = a->as_u16[i] == 0;
100 if (is_zero && i_first_zero >= ARRAY_LEN (a->as_u16))
106 if ((!is_zero && n_zeros > max_n_zeros)
107 || (i + 1 >= ARRAY_LEN (a->as_u16) && n_zeros > max_n_zeros))
109 i_max_n_zero = i_first_zero;
110 max_n_zeros = n_zeros;
111 i_first_zero = ARRAY_LEN (a->as_u16);
116 last_double_colon = 0;
117 for (i = 0; i < ARRAY_LEN (a->as_u16); i++)
119 if (i == i_max_n_zero && max_n_zeros > 1)
121 s = format (s, "::");
122 i += max_n_zeros - 1;
123 last_double_colon = 1;
127 s = format (s, "%s%x",
128 (last_double_colon || i == 0) ? "" : ":",
129 clib_net_to_host_u16 (a->as_u16[i]));
130 last_double_colon = 0;
137 /* Format an IP46 address. */
139 format_ip46_address (u8 * s, va_list * args)
141 ip46_address_t *ip46 = va_arg (*args, ip46_address_t *);
142 ip46_type_t type = va_arg (*args, ip46_type_t);
148 is_ip4 = ip46_address_is_ip4 (ip46);
159 format (s, "%U", format_ip4_address, &ip46->ip4) :
160 format (s, "%U", format_ip6_address, &ip46->ip6);
164 * VPPCOM Utility Functions
168 vcl_send_session_listen (vcl_worker_t * wrk, vcl_session_t * s)
170 app_session_evt_t _app_evt, *app_evt = &_app_evt;
171 session_listen_msg_t *mp;
174 mq = vcl_worker_ctrl_mq (wrk);
175 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN);
176 mp = (session_listen_msg_t *) app_evt->evt->data;
177 memset (mp, 0, sizeof (*mp));
178 mp->client_index = wrk->api_client_handle;
179 mp->context = s->session_index;
180 mp->wrk_index = wrk->vpp_wrk_index;
181 mp->is_ip4 = s->transport.is_ip4;
182 clib_memcpy_fast (&mp->ip, &s->transport.lcl_ip, sizeof (mp->ip));
183 mp->port = s->transport.lcl_port;
184 mp->proto = s->session_type;
185 mp->ckpair_index = s->ckpair_index;
186 if (s->flags & VCL_SESSION_F_CONNECTED)
187 mp->flags = TRANSPORT_CFG_F_CONNECTED;
188 app_send_ctrl_evt_to_vpp (mq, app_evt);
192 vcl_send_session_connect (vcl_worker_t * wrk, vcl_session_t * s)
194 app_session_evt_t _app_evt, *app_evt = &_app_evt;
195 session_connect_msg_t *mp;
198 mq = vcl_worker_ctrl_mq (wrk);
199 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT);
200 mp = (session_connect_msg_t *) app_evt->evt->data;
201 memset (mp, 0, sizeof (*mp));
202 mp->client_index = wrk->api_client_handle;
203 mp->context = s->session_index;
204 mp->wrk_index = wrk->vpp_wrk_index;
205 mp->is_ip4 = s->transport.is_ip4;
206 mp->parent_handle = s->parent_handle;
207 clib_memcpy_fast (&mp->ip, &s->transport.rmt_ip, sizeof (mp->ip));
208 clib_memcpy_fast (&mp->lcl_ip, &s->transport.lcl_ip, sizeof (mp->lcl_ip));
209 mp->port = s->transport.rmt_port;
210 mp->lcl_port = s->transport.lcl_port;
211 mp->proto = s->session_type;
212 mp->ckpair_index = s->ckpair_index;
213 if (s->flags & VCL_SESSION_F_CONNECTED)
214 mp->flags |= TRANSPORT_CFG_F_CONNECTED;
215 app_send_ctrl_evt_to_vpp (mq, app_evt);
219 vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s)
221 app_session_evt_t _app_evt, *app_evt = &_app_evt;
222 session_unlisten_msg_t *mp;
225 mq = vcl_worker_ctrl_mq (wrk);
226 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN);
227 mp = (session_unlisten_msg_t *) app_evt->evt->data;
228 memset (mp, 0, sizeof (*mp));
229 mp->client_index = wrk->api_client_handle;
230 mp->wrk_index = wrk->vpp_wrk_index;
231 mp->handle = s->vpp_handle;
232 mp->context = wrk->wrk_index;
233 app_send_ctrl_evt_to_vpp (mq, app_evt);
237 vcl_send_session_disconnect (vcl_worker_t * wrk, vcl_session_t * s)
239 app_session_evt_t _app_evt, *app_evt = &_app_evt;
240 session_disconnect_msg_t *mp;
243 /* Send to thread that owns the session */
245 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT);
246 mp = (session_disconnect_msg_t *) app_evt->evt->data;
247 memset (mp, 0, sizeof (*mp));
248 mp->client_index = wrk->api_client_handle;
249 mp->handle = s->vpp_handle;
250 app_send_ctrl_evt_to_vpp (mq, app_evt);
254 vcl_send_app_detach (vcl_worker_t * wrk)
256 app_session_evt_t _app_evt, *app_evt = &_app_evt;
257 session_app_detach_msg_t *mp;
260 mq = vcl_worker_ctrl_mq (wrk);
261 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_DETACH);
262 mp = (session_app_detach_msg_t *) app_evt->evt->data;
263 memset (mp, 0, sizeof (*mp));
264 mp->client_index = wrk->api_client_handle;
265 app_send_ctrl_evt_to_vpp (mq, app_evt);
269 vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context,
270 session_handle_t handle, int retval)
272 app_session_evt_t _app_evt, *app_evt = &_app_evt;
273 session_accepted_reply_msg_t *rmp;
274 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_ACCEPTED_REPLY);
275 rmp = (session_accepted_reply_msg_t *) app_evt->evt->data;
276 rmp->handle = handle;
277 rmp->context = context;
278 rmp->retval = retval;
279 app_send_ctrl_evt_to_vpp (mq, app_evt);
283 vcl_send_session_disconnected_reply (vcl_worker_t * wrk, vcl_session_t * s,
286 app_session_evt_t _app_evt, *app_evt = &_app_evt;
287 session_disconnected_reply_msg_t *rmp;
288 app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
289 SESSION_CTRL_EVT_DISCONNECTED_REPLY);
290 rmp = (session_disconnected_reply_msg_t *) app_evt->evt->data;
291 rmp->handle = s->vpp_handle;
292 rmp->context = wrk->api_client_handle;
293 rmp->retval = retval;
294 app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
298 vcl_send_session_reset_reply (vcl_worker_t * wrk, vcl_session_t * s,
301 app_session_evt_t _app_evt, *app_evt = &_app_evt;
302 session_reset_reply_msg_t *rmp;
303 app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
304 SESSION_CTRL_EVT_RESET_REPLY);
305 rmp = (session_reset_reply_msg_t *) app_evt->evt->data;
306 rmp->handle = s->vpp_handle;
307 rmp->context = wrk->api_client_handle;
308 rmp->retval = retval;
309 app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
313 vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
316 app_session_evt_t _app_evt, *app_evt = &_app_evt;
317 session_worker_update_msg_t *mp;
319 app_alloc_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt,
320 SESSION_CTRL_EVT_WORKER_UPDATE);
321 mp = (session_worker_update_msg_t *) app_evt->evt->data;
322 mp->client_index = wrk->api_client_handle;
323 mp->handle = s->vpp_handle;
324 mp->req_wrk_index = wrk->vpp_wrk_index;
325 mp->wrk_index = wrk_index;
326 app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
330 vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len)
332 app_session_evt_t _app_evt, *app_evt = &_app_evt;
333 session_app_wrk_rpc_msg_t *mp;
334 vcl_worker_t *dst_wrk, *wrk;
338 if (data_len > sizeof (mp->data))
341 clib_spinlock_lock (&vcm->workers_lock);
343 dst_wrk = vcl_worker_get_if_valid (dst_wrk_index);
347 wrk = vcl_worker_get_current ();
348 mq = vcl_worker_ctrl_mq (wrk);
349 app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_WRK_RPC);
350 mp = (session_app_wrk_rpc_msg_t *) app_evt->evt->data;
351 mp->client_index = wrk->api_client_handle;
352 mp->wrk_index = dst_wrk->vpp_wrk_index;
353 clib_memcpy (mp->data, data, data_len);
354 app_send_ctrl_evt_to_vpp (mq, app_evt);
358 clib_spinlock_unlock (&vcm->workers_lock);
363 vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp,
366 vcl_session_t *session, *listen_session;
369 session = vcl_session_alloc (wrk);
371 listen_session = vcl_session_get (wrk, ls_index);
372 if (listen_session->vpp_handle != mp->listener_handle)
374 VDBG (0, "ERROR: listener handle %lu does not match session %u",
375 mp->listener_handle, ls_index);
379 if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo,
381 mp->vpp_event_queue_address, 0, session))
383 VDBG (0, "failed to attach fifos for %u", session->session_index);
387 session->vpp_handle = mp->handle;
388 session->session_state = VCL_STATE_READY;
389 session->transport.rmt_port = mp->rmt.port;
390 session->transport.is_ip4 = mp->rmt.is_ip4;
391 clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip,
392 sizeof (ip46_address_t));
394 vcl_session_table_add_vpp_handle (wrk, mp->handle, session->session_index);
395 session->transport.lcl_port = listen_session->transport.lcl_port;
396 session->transport.lcl_ip = listen_session->transport.lcl_ip;
397 session->session_type = listen_session->session_type;
398 session->is_dgram = vcl_proto_is_dgram (session->session_type);
399 session->listener_index = listen_session->session_index;
400 listen_session->n_accepted_sessions++;
402 VDBG (1, "session %u [0x%llx]: client accept request from %s address %U"
403 " port %d queue %p!", session->session_index, mp->handle,
404 mp->rmt.is_ip4 ? "IPv4" : "IPv6", format_ip46_address, &mp->rmt.ip,
405 mp->rmt.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
406 clib_net_to_host_u16 (mp->rmt.port), session->vpp_evt_q);
407 vcl_evt (VCL_EVT_ACCEPT, session, listen_session, session_index);
409 vcl_send_session_accepted_reply (session->vpp_evt_q, mp->context,
410 session->vpp_handle, 0);
412 return session->session_index;
415 vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0),
416 mp->vpp_event_queue_address, mp->mq_index, &evt_q);
417 vcl_send_session_accepted_reply (evt_q, mp->context, mp->handle,
418 VNET_API_ERROR_INVALID_ARGUMENT);
419 vcl_session_free (wrk, session);
420 return VCL_INVALID_SESSION_INDEX;
424 vcl_session_connected_handler (vcl_worker_t * wrk,
425 session_connected_msg_t * mp)
427 vcl_session_t *session = 0;
430 session_index = mp->context;
431 session = vcl_session_get (wrk, session_index);
434 VDBG (0, "ERROR: vpp handle 0x%llx has no session index (%u)!",
435 mp->handle, session_index);
436 return VCL_INVALID_SESSION_INDEX;
440 VDBG (0, "ERROR: session index %u: connect failed! %U",
441 session_index, format_session_error, mp->retval);
442 session->session_state = VCL_STATE_DETACHED;
443 session->vpp_handle = mp->handle;
444 return session_index;
447 session->vpp_handle = mp->handle;
449 if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo,
451 mp->vpp_event_queue_address, 0, session))
453 VDBG (0, "failed to attach fifos for %u", session->session_index);
454 session->session_state = VCL_STATE_DETACHED;
455 vcl_send_session_disconnect (wrk, session);
456 return session_index;
461 if (vcl_segment_attach_session (mp->ct_segment_handle, mp->ct_rx_fifo,
462 mp->ct_tx_fifo, (uword) ~0, 1, session))
464 VDBG (0, "failed to attach ct fifos for %u", session->session_index);
465 session->session_state = VCL_STATE_DETACHED;
466 vcl_send_session_disconnect (wrk, session);
467 return session_index;
471 session->transport.is_ip4 = mp->lcl.is_ip4;
472 clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip,
473 sizeof (session->transport.lcl_ip));
474 session->transport.lcl_port = mp->lcl.port;
476 /* Application closed session before connect reply */
477 if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK)
478 && session->session_state == VCL_STATE_CLOSED)
479 vcl_send_session_disconnect (wrk, session);
481 session->session_state = VCL_STATE_READY;
483 /* Add it to lookup table */
484 vcl_session_table_add_vpp_handle (wrk, mp->handle, session_index);
486 VDBG (1, "session %u [0x%llx] connected! rx_fifo %p, refcnt %d, tx_fifo %p,"
487 " refcnt %d", session_index, mp->handle, session->rx_fifo,
488 session->rx_fifo->refcnt, session->tx_fifo, session->tx_fifo->refcnt);
490 return session_index;
494 vcl_flag_accepted_session (vcl_session_t * session, u64 handle, u32 flags)
496 vcl_session_msg_t *accepted_msg;
499 for (i = 0; i < vec_len (session->accept_evts_fifo); i++)
501 accepted_msg = &session->accept_evts_fifo[i];
502 if (accepted_msg->accepted_msg.handle == handle)
504 accepted_msg->flags |= flags;
512 vcl_session_reset_handler (vcl_worker_t * wrk,
513 session_reset_msg_t * reset_msg)
515 vcl_session_t *session;
518 sid = vcl_session_index_from_vpp_handle (wrk, reset_msg->handle);
519 session = vcl_session_get (wrk, sid);
522 VDBG (0, "request to reset unknown handle 0x%llx", reset_msg->handle);
523 return VCL_INVALID_SESSION_INDEX;
526 /* Caught a reset before actually accepting the session */
527 if (session->session_state == VCL_STATE_LISTEN)
530 if (!vcl_flag_accepted_session (session, reset_msg->handle,
531 VCL_ACCEPTED_F_RESET))
532 VDBG (0, "session was not accepted!");
533 return VCL_INVALID_SESSION_INDEX;
536 if (session->session_state != VCL_STATE_CLOSED)
537 session->session_state = VCL_STATE_DISCONNECT;
538 VDBG (0, "reset session %u [0x%llx]", sid, reset_msg->handle);
543 vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp)
545 vcl_session_t *session;
546 u32 sid = mp->context;
548 session = vcl_session_get (wrk, sid);
551 VERR ("session %u [0x%llx]: bind failed: %U", sid, mp->handle,
552 format_session_error, mp->retval);
555 session->session_state = VCL_STATE_DETACHED;
556 session->vpp_handle = mp->handle;
561 VDBG (0, "ERROR: session %u [0x%llx]: Invalid session index!",
563 return VCL_INVALID_SESSION_INDEX;
567 session->vpp_handle = mp->handle;
568 session->transport.is_ip4 = mp->lcl_is_ip4;
569 clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip,
570 sizeof (ip46_address_t));
571 session->transport.lcl_port = mp->lcl_port;
572 vcl_session_table_add_listener (wrk, mp->handle, sid);
573 session->session_state = VCL_STATE_LISTEN;
575 if (vcl_session_is_cl (session))
577 if (vcl_segment_attach_session (mp->segment_handle, mp->rx_fifo,
578 mp->tx_fifo, mp->vpp_evt_q, 0, session))
580 VDBG (0, "failed to attach fifos for %u", session->session_index);
581 session->session_state = VCL_STATE_DETACHED;
582 return VCL_INVALID_SESSION_INDEX;
586 VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle);
591 vcl_session_unlisten_reply_handler (vcl_worker_t * wrk, void *data)
593 session_unlisten_reply_msg_t *mp = (session_unlisten_reply_msg_t *) data;
596 s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
599 VDBG (0, "Unlisten reply with wrong handle %llx", mp->handle);
602 if (s->session_state != VCL_STATE_DISCONNECT)
604 /* Connected udp listener */
605 if (s->session_type == VPPCOM_PROTO_UDP
606 && s->session_state == VCL_STATE_CLOSED)
609 VDBG (0, "Unlisten session in wrong state %llx", mp->handle);
614 VDBG (0, "ERROR: session %u [0xllx]: unlisten failed: %U",
615 s->session_index, mp->handle, format_session_error, mp->retval);
617 if (mp->context != wrk->wrk_index)
618 VDBG (0, "wrong context");
620 vcl_session_table_del_vpp_handle (wrk, mp->handle);
621 vcl_session_free (wrk, s);
625 vcl_session_migrated_handler (vcl_worker_t * wrk, void *data)
627 session_migrated_msg_t *mp = (session_migrated_msg_t *) data;
631 s = vcl_session_get_w_vpp_handle (wrk, mp->handle);
634 VDBG (0, "Migrated notification with wrong handle %llx", mp->handle);
638 fs_index = vcl_segment_table_lookup (mp->segment_handle);
639 if (fs_index == VCL_INVALID_SEGMENT_INDEX)
641 VDBG (0, "segment for session %u is not mounted!", s->session_index);
642 s->session_state = VCL_STATE_DETACHED;
646 s->vpp_handle = mp->new_handle;
648 vcl_segment_attach_mq (vcl_vpp_worker_segment_handle (0), mp->vpp_evt_q,
649 mp->vpp_thread_index, &s->vpp_evt_q);
651 vcl_session_table_del_vpp_handle (wrk, mp->handle);
652 vcl_session_table_add_vpp_handle (wrk, mp->new_handle, s->session_index);
654 /* Generate new tx event if we have outstanding data */
655 if (svm_fifo_has_event (s->tx_fifo))
656 app_send_io_evt_to_vpp (s->vpp_evt_q,
657 s->tx_fifo->shr->master_session_index,
658 SESSION_IO_EVT_TX, SVM_Q_WAIT);
660 VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle,
661 mp->vpp_thread_index, mp->new_handle);
664 static vcl_session_t *
665 vcl_session_accepted (vcl_worker_t * wrk, session_accepted_msg_t * msg)
667 vcl_session_msg_t *vcl_msg;
668 vcl_session_t *session;
670 session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
671 if (PREDICT_FALSE (session != 0))
672 VWRN ("session overlap handle %lu state %u!", msg->handle,
673 session->session_state);
675 session = vcl_session_table_lookup_listener (wrk, msg->listener_handle);
678 VERR ("couldn't find listen session: listener handle %llx",
679 msg->listener_handle);
683 clib_fifo_add2 (session->accept_evts_fifo, vcl_msg);
685 vcl_msg->accepted_msg = *msg;
686 /* Session handle points to listener until fully accepted by app */
687 vcl_session_table_add_vpp_handle (wrk, msg->handle, session->session_index);
692 static vcl_session_t *
693 vcl_session_disconnected_handler (vcl_worker_t * wrk,
694 session_disconnected_msg_t * msg)
696 vcl_session_t *session;
698 session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
701 VDBG (0, "request to disconnect unknown handle 0x%llx", msg->handle);
705 /* Late disconnect notification on a session that has been closed */
706 if (session->session_state == VCL_STATE_CLOSED)
709 /* Caught a disconnect before actually accepting the session */
710 if (session->session_state == VCL_STATE_LISTEN)
712 if (!vcl_flag_accepted_session (session, msg->handle,
713 VCL_ACCEPTED_F_CLOSED))
714 VDBG (0, "session was not accepted!");
718 /* If not already reset change state */
719 if (session->session_state != VCL_STATE_DISCONNECT)
720 session->session_state = VCL_STATE_VPP_CLOSING;
726 vppcom_session_disconnect (u32 session_handle)
728 vcl_worker_t *wrk = vcl_worker_get_current ();
729 vcl_session_t *session, *listen_session;
730 vcl_session_state_t state;
733 session = vcl_session_get_w_handle (wrk, session_handle);
735 return VPPCOM_EBADFD;
737 vpp_handle = session->vpp_handle;
738 state = session->session_state;
740 VDBG (1, "session %u [0x%llx] state 0x%x (%s)", session->session_index,
741 vpp_handle, state, vppcom_session_state_str (state));
743 if (PREDICT_FALSE (state == VCL_STATE_LISTEN))
745 VDBG (0, "ERROR: Cannot disconnect a listen socket!");
746 return VPPCOM_EBADFD;
749 if (state == VCL_STATE_VPP_CLOSING)
751 vcl_send_session_disconnected_reply (wrk, session, 0);
752 VDBG (1, "session %u [0x%llx]: sending disconnect REPLY...",
753 session->session_index, vpp_handle);
757 /* Session doesn't have an event queue yet. Probably a non-blocking
758 * connect. Wait for the reply */
759 if (PREDICT_FALSE (!session->vpp_evt_q))
762 VDBG (1, "session %u [0x%llx]: sending disconnect...",
763 session->session_index, vpp_handle);
764 vcl_send_session_disconnect (wrk, session);
767 if (session->listener_index != VCL_INVALID_SESSION_INDEX)
769 listen_session = vcl_session_get (wrk, session->listener_index);
770 listen_session->n_accepted_sessions--;
777 vcl_session_cleanup_handler (vcl_worker_t * wrk, void *data)
779 session_cleanup_msg_t *msg;
780 vcl_session_t *session;
782 msg = (session_cleanup_msg_t *) data;
783 session = vcl_session_get_w_vpp_handle (wrk, msg->handle);
786 VDBG (0, "disconnect confirmed for unknown handle 0x%llx", msg->handle);
790 if (msg->type == SESSION_CLEANUP_TRANSPORT)
792 /* Transport was cleaned up before we confirmed close. Probably the
793 * app is still waiting for some data that cannot be delivered.
794 * Confirm close to make sure everything is cleaned up.
795 * Move to undetermined state to ensure that the session is not
796 * removed before both vpp and the app cleanup.
797 * - If the app closes first, the session is moved to CLOSED state
798 * and the session cleanup notification from vpp removes the
800 * - If vpp cleans up the session first, the session is moved to
801 * DETACHED state lower and subsequently the close from the app
804 if (session->session_state == VCL_STATE_VPP_CLOSING)
806 vppcom_session_disconnect (vcl_session_handle (session));
807 session->session_state = VCL_STATE_UPDATED;
809 else if (session->session_state == VCL_STATE_DISCONNECT)
811 vcl_send_session_reset_reply (wrk, session, 0);
812 session->session_state = VCL_STATE_UPDATED;
817 vcl_session_table_del_vpp_handle (wrk, msg->handle);
818 /* Should not happen. App did not close the connection so don't free it. */
819 if (session->session_state != VCL_STATE_CLOSED)
821 VDBG (0, "app did not close session %d", session->session_index);
822 session->session_state = VCL_STATE_DETACHED;
823 session->vpp_handle = VCL_INVALID_SESSION_HANDLE;
826 vcl_session_free (wrk, session);
830 vcl_session_req_worker_update_handler (vcl_worker_t * wrk, void *data)
832 session_req_worker_update_msg_t *msg;
835 msg = (session_req_worker_update_msg_t *) data;
836 s = vcl_session_get_w_vpp_handle (wrk, msg->session_handle);
840 vec_add1 (wrk->pending_session_wrk_updates, s->session_index);
844 vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
846 session_worker_update_reply_msg_t *msg;
849 msg = (session_worker_update_reply_msg_t *) data;
850 s = vcl_session_get_w_vpp_handle (wrk, msg->handle);
853 VDBG (0, "unknown handle 0x%llx", msg->handle);
859 if (vcl_segment_attach_session (msg->segment_handle, msg->rx_fifo,
860 msg->tx_fifo, (uword) ~0, 0, s))
862 VDBG (0, "failed to attach fifos for %u", s->session_index);
866 s->session_state = VCL_STATE_UPDATED;
868 VDBG (0, "session %u[0x%llx] moved to worker %u", s->session_index,
869 s->vpp_handle, wrk->wrk_index);
873 vcl_api_recv_fd (vcl_worker_t * wrk, int *fds, int n_fds)
876 if (vcm->cfg.vpp_app_socket_api)
877 return vcl_sapi_recv_fds (wrk, fds, n_fds);
879 return vcl_bapi_recv_fds (wrk, fds, n_fds);
883 vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
885 ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM;
886 session_app_add_segment_msg_t *msg;
890 msg = (session_app_add_segment_msg_t *) data;
894 vcl_api_recv_fd (wrk, &fd, 1);
895 seg_type = SSVM_SEGMENT_MEMFD;
898 segment_handle = msg->segment_handle;
899 if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
901 clib_warning ("invalid segment handle");
905 if (vcl_segment_attach (segment_handle, (char *) msg->segment_name,
908 VDBG (0, "vcl_segment_attach ('%s') failed", msg->segment_name);
912 VDBG (1, "mapped new segment '%s' size %d", msg->segment_name,
917 vcl_session_app_del_segment_handler (vcl_worker_t * wrk, void *data)
919 session_app_del_segment_msg_t *msg = (session_app_del_segment_msg_t *) data;
920 vcl_segment_detach (msg->segment_handle);
921 VDBG (1, "Unmapped segment: %d", msg->segment_handle);
925 vcl_worker_rpc_handler (vcl_worker_t * wrk, void *data)
927 if (!vcm->wrk_rpc_fn)
930 (vcm->wrk_rpc_fn) (((session_app_wrk_rpc_msg_t *) data)->data);
934 vcl_handle_mq_event (vcl_worker_t * wrk, session_event_t * e)
936 session_disconnected_msg_t *disconnected_msg;
937 session_connected_msg_t *connected_msg;
938 session_reset_msg_t *reset_msg;
939 session_event_t *ecpy;
943 switch (e->event_type)
945 case SESSION_IO_EVT_RX:
946 case SESSION_IO_EVT_TX:
947 s = vcl_session_get (wrk, e->session_index);
948 if (!s || !vcl_session_is_open (s))
950 vec_add1 (wrk->unhandled_evts_vector, *e);
952 case SESSION_CTRL_EVT_BOUND:
953 /* We can only wait for only one listen so not postponed */
954 vcl_session_bound_handler (wrk, (session_bound_msg_t *) e->data);
956 case SESSION_CTRL_EVT_ACCEPTED:
957 s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
958 if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
960 vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
963 ecpy->session_index = s->session_index;
966 case SESSION_CTRL_EVT_CONNECTED:
967 connected_msg = (session_connected_msg_t *) e->data;
968 sid = vcl_session_connected_handler (wrk, connected_msg);
969 if (!(s = vcl_session_get (wrk, sid)))
971 if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
973 vec_add2 (wrk->unhandled_evts_vector, ecpy, 1);
976 ecpy->session_index = s->session_index;
979 case SESSION_CTRL_EVT_DISCONNECTED:
980 disconnected_msg = (session_disconnected_msg_t *) e->data;
981 if (!(s = vcl_session_get_w_vpp_handle (wrk, disconnected_msg->handle)))
983 if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
985 vec_add1 (wrk->unhandled_evts_vector, *e);
988 if (!(s = vcl_session_disconnected_handler (wrk, disconnected_msg)))
990 VDBG (0, "disconnected session %u [0x%llx]", s->session_index,
993 case SESSION_CTRL_EVT_RESET:
994 reset_msg = (session_reset_msg_t *) e->data;
995 if (!(s = vcl_session_get_w_vpp_handle (wrk, reset_msg->handle)))
997 if (vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
999 vec_add1 (wrk->unhandled_evts_vector, *e);
1002 vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
1004 case SESSION_CTRL_EVT_UNLISTEN_REPLY:
1005 vcl_session_unlisten_reply_handler (wrk, e->data);
1007 case SESSION_CTRL_EVT_MIGRATED:
1008 vcl_session_migrated_handler (wrk, e->data);
1010 case SESSION_CTRL_EVT_CLEANUP:
1011 vcl_session_cleanup_handler (wrk, e->data);
1013 case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
1014 vcl_session_req_worker_update_handler (wrk, e->data);
1016 case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
1017 vcl_session_worker_update_reply_handler (wrk, e->data);
1019 case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
1020 vcl_session_app_add_segment_handler (wrk, e->data);
1022 case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
1023 vcl_session_app_del_segment_handler (wrk, e->data);
1025 case SESSION_CTRL_EVT_APP_WRK_RPC:
1026 vcl_worker_rpc_handler (wrk, e->data);
1029 clib_warning ("unhandled %u", e->event_type);
1035 vppcom_wait_for_session_state_change (u32 session_index,
1036 vcl_session_state_t state,
1039 vcl_worker_t *wrk = vcl_worker_get_current ();
1040 f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
1041 vcl_session_t *volatile session;
1042 svm_msg_q_msg_t msg;
1047 session = vcl_session_get (wrk, session_index);
1048 if (PREDICT_FALSE (!session))
1050 return VPPCOM_EBADFD;
1052 if (session->session_state == state)
1056 if (session->session_state == VCL_STATE_DETACHED)
1058 return VPPCOM_ECONNREFUSED;
1061 if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0))
1066 e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
1067 vcl_handle_mq_event (wrk, e);
1068 svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1070 while (clib_time_now (&wrk->clib_time) < timeout);
1072 VDBG (0, "timeout waiting for state 0x%x (%s)", state,
1073 vppcom_session_state_str (state));
1074 vcl_evt (VCL_EVT_SESSION_TIMEOUT, session, session_state);
1076 return VPPCOM_ETIMEDOUT;
1080 vcl_handle_pending_wrk_updates (vcl_worker_t * wrk)
1082 vcl_session_state_t state;
1086 if (PREDICT_TRUE (vec_len (wrk->pending_session_wrk_updates) == 0))
1089 vec_foreach (sip, wrk->pending_session_wrk_updates)
1091 s = vcl_session_get (wrk, *sip);
1092 vcl_send_session_worker_update (wrk, s, wrk->wrk_index);
1093 state = s->session_state;
1094 vppcom_wait_for_session_state_change (s->session_index, VCL_STATE_UPDATED,
1096 s->session_state = state;
1098 vec_reset_length (wrk->pending_session_wrk_updates);
1102 vcl_flush_mq_events (void)
1104 vcl_worker_t *wrk = vcl_worker_get_current ();
1105 svm_msg_q_msg_t *msg;
1110 mq = wrk->app_event_queue;
1111 svm_msg_q_lock (mq);
1112 vcl_mq_dequeue_batch (wrk, mq, ~0);
1113 svm_msg_q_unlock (mq);
1115 for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
1117 msg = vec_elt_at_index (wrk->mq_msg_vector, i);
1118 e = svm_msg_q_msg_data (mq, msg);
1119 vcl_handle_mq_event (wrk, e);
1120 svm_msg_q_free_msg (mq, msg);
1122 vec_reset_length (wrk->mq_msg_vector);
1123 vcl_handle_pending_wrk_updates (wrk);
1127 vppcom_session_unbind (u32 session_handle)
1129 vcl_worker_t *wrk = vcl_worker_get_current ();
1130 session_accepted_msg_t *accepted_msg;
1131 vcl_session_t *session = 0;
1132 vcl_session_msg_t *evt;
1134 session = vcl_session_get_w_handle (wrk, session_handle);
1136 return VPPCOM_EBADFD;
1138 /* Flush pending accept events, if any */
1139 while (clib_fifo_elts (session->accept_evts_fifo))
1141 clib_fifo_sub2 (session->accept_evts_fifo, evt);
1142 accepted_msg = &evt->accepted_msg;
1143 vcl_session_table_del_vpp_handle (wrk, accepted_msg->handle);
1144 vcl_send_session_accepted_reply (session->vpp_evt_q,
1145 accepted_msg->context,
1146 accepted_msg->handle, -1);
1148 clib_fifo_free (session->accept_evts_fifo);
1150 vcl_send_session_unlisten (wrk, session);
1152 VDBG (1, "session %u [0x%llx]: sending unbind!", session->session_index,
1153 session->vpp_handle);
1154 vcl_evt (VCL_EVT_UNBIND, session);
1156 session->vpp_handle = ~0;
1157 session->session_state = VCL_STATE_DISCONNECT;
1165 * Notify vpp of the disconnect and mark the worker as free. If we're the
1166 * last worker, do a full cleanup otherwise, since we're probably a forked
1167 * child, avoid syscalls as much as possible. We might've lost privileges.
1170 vppcom_app_exit (void)
1172 if (!pool_elts (vcm->workers))
1174 vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
1175 vcl_set_worker_index (~0);
1176 vcl_elog_stop (vcm);
1180 vcl_api_attach (void)
1182 if (vcm->cfg.vpp_app_socket_api)
1183 return vcl_sapi_attach ();
1185 return vcl_bapi_attach ();
1189 vcl_api_detach (vcl_worker_t * wrk)
1191 vcl_send_app_detach (wrk);
1193 if (vcm->cfg.vpp_app_socket_api)
1194 return vcl_sapi_detach (wrk);
1196 return vcl_bapi_disconnect_from_vpp ();
1200 * VPPCOM Public API functions
1203 vppcom_app_create (const char *app_name)
1205 vppcom_cfg_t *vcl_cfg = &vcm->cfg;
1210 VDBG (1, "already initialized");
1211 return VPPCOM_EEXIST;
1215 vppcom_cfg (&vcm->cfg);
1216 vcl_cfg = &vcm->cfg;
1218 vcm->main_cpu = pthread_self ();
1219 vcm->main_pid = getpid ();
1220 vcm->app_name = format (0, "%s", app_name);
1221 fifo_segment_main_init (&vcm->segment_main, vcl_cfg->segment_baseva,
1222 20 /* timeout in secs */ );
1223 pool_alloc (vcm->workers, vcl_cfg->max_workers);
1224 clib_spinlock_init (&vcm->workers_lock);
1225 clib_rwlock_init (&vcm->segment_table_lock);
1226 atexit (vppcom_app_exit);
1227 vcl_elog_init (vcm);
1229 /* Allocate default worker */
1230 vcl_worker_alloc_and_init ();
1232 if ((rv = vcl_api_attach ()))
1235 VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
1236 vcm->workers[0].api_client_handle, vcm->workers[0].api_client_handle);
1242 vppcom_app_destroy (void)
1244 vcl_worker_t *wrk, *current_wrk;
1247 if (!pool_elts (vcm->workers))
1250 vcl_evt (VCL_EVT_DETACH, vcm);
1252 current_wrk = vcl_worker_get_current ();
1255 pool_foreach (wrk, vcm->workers) {
1256 if (current_wrk != wrk)
1257 vcl_worker_cleanup (wrk, 0 /* notify vpp */ );
1261 vcl_api_detach (current_wrk);
1262 vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
1264 vcl_elog_stop (vcm);
1267 * Free the heap and fix vcm
1269 heap = clib_mem_get_heap ();
1270 munmap (clib_mem_get_heap_base (heap), clib_mem_get_heap_size (heap));
1272 vcm = &_vppcom_main;
1277 vppcom_session_create (u8 proto, u8 is_nonblocking)
1279 vcl_worker_t *wrk = vcl_worker_get_current ();
1280 vcl_session_t *session;
1282 session = vcl_session_alloc (wrk);
1284 session->session_type = proto;
1285 session->session_state = VCL_STATE_CLOSED;
1286 session->vpp_handle = ~0;
1287 session->ckpair_index = ~0;
1288 session->is_dgram = vcl_proto_is_dgram (proto);
1291 vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
1293 vcl_evt (VCL_EVT_CREATE, session, session_type, session->session_state,
1294 is_nonblocking, session_index);
1296 VDBG (0, "created session %u", session->session_index);
1298 return vcl_session_handle (session);
1302 vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
1303 vcl_session_handle_t sh, u8 do_disconnect)
1307 VDBG (1, "session %u [0x%llx] closing", s->session_index, s->vpp_handle);
1309 if (s->flags & VCL_SESSION_F_IS_VEP)
1311 u32 next_sh = s->vep.next_sh;
1312 while (next_sh != ~0)
1314 rv = vppcom_epoll_ctl (sh, EPOLL_CTL_DEL, next_sh, 0);
1315 if (PREDICT_FALSE (rv < 0))
1316 VDBG (0, "vpp handle 0x%llx, sh %u: EPOLL_CTL_DEL vep_idx %u"
1317 " failed! rv %d (%s)", s->vpp_handle, next_sh,
1318 s->vep.vep_sh, rv, vppcom_retval_str (rv));
1319 next_sh = s->vep.next_sh;
1324 if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
1326 rv = vppcom_epoll_ctl (s->vep.vep_sh, EPOLL_CTL_DEL, sh, 0);
1328 VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
1329 "failed! rv %d (%s)", s->session_index, s->vpp_handle,
1330 s->vep.vep_sh, rv, vppcom_retval_str (rv));
1335 VDBG (1, "session %u [0x%llx] disconnect skipped",
1336 s->session_index, s->vpp_handle);
1340 if (s->session_state == VCL_STATE_LISTEN)
1342 rv = vppcom_session_unbind (sh);
1343 if (PREDICT_FALSE (rv < 0))
1344 VDBG (0, "session %u [0x%llx]: listener unbind failed! "
1345 "rv %d (%s)", s->session_index, s->vpp_handle, rv,
1346 vppcom_retval_str (rv));
1349 else if (vcl_session_is_ready (s)
1350 || (vcl_session_is_connectable_listener (wrk, s)))
1352 rv = vppcom_session_disconnect (sh);
1353 if (PREDICT_FALSE (rv < 0))
1354 VDBG (0, "ERROR: session %u [0x%llx]: disconnect failed!"
1355 " rv %d (%s)", s->session_index, s->vpp_handle,
1356 rv, vppcom_retval_str (rv));
1358 else if (s->session_state == VCL_STATE_DISCONNECT)
1360 vcl_send_session_reset_reply (wrk, s, 0);
1362 else if (s->session_state == VCL_STATE_DETACHED)
1364 /* Should not happen. VPP cleaned up before app confirmed close */
1365 VDBG (0, "vpp freed session %d before close", s->session_index);
1369 s->session_state = VCL_STATE_CLOSED;
1371 /* Session is removed only after vpp confirms the disconnect */
1375 vcl_session_table_del_vpp_handle (wrk, s->vpp_handle);
1377 vcl_session_free (wrk, s);
1378 vcl_evt (VCL_EVT_CLOSE, s, rv);
1384 vppcom_session_close (uint32_t session_handle)
1386 vcl_worker_t *wrk = vcl_worker_get_current ();
1387 vcl_session_t *session;
1389 session = vcl_session_get_w_handle (wrk, session_handle);
1391 return VPPCOM_EBADFD;
1392 return vcl_session_cleanup (wrk, session, session_handle,
1393 1 /* do_disconnect */ );
1397 vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep)
1399 vcl_worker_t *wrk = vcl_worker_get_current ();
1400 vcl_session_t *session = 0;
1403 return VPPCOM_EINVAL;
1405 session = vcl_session_get_w_handle (wrk, session_handle);
1407 return VPPCOM_EBADFD;
1409 if (session->flags & VCL_SESSION_F_IS_VEP)
1411 VDBG (0, "ERROR: cannot bind to epoll session %u!",
1412 session->session_index);
1413 return VPPCOM_EBADFD;
1416 session->transport.is_ip4 = ep->is_ip4;
1418 clib_memcpy_fast (&session->transport.lcl_ip.ip4, ep->ip,
1419 sizeof (ip4_address_t));
1421 clib_memcpy_fast (&session->transport.lcl_ip.ip6, ep->ip,
1422 sizeof (ip6_address_t));
1423 session->transport.lcl_port = ep->port;
1425 VDBG (0, "session %u handle %u: binding to local %s address %U port %u, "
1426 "proto %s", session->session_index, session_handle,
1427 session->transport.is_ip4 ? "IPv4" : "IPv6",
1428 format_ip46_address, &session->transport.lcl_ip,
1429 session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1430 clib_net_to_host_u16 (session->transport.lcl_port),
1431 vppcom_proto_str (session->session_type));
1432 vcl_evt (VCL_EVT_BIND, session);
1434 if (session->session_type == VPPCOM_PROTO_UDP)
1435 vppcom_session_listen (session_handle, 10);
1441 vppcom_session_listen (uint32_t listen_sh, uint32_t q_len)
1443 vcl_worker_t *wrk = vcl_worker_get_current ();
1444 vcl_session_t *listen_session = 0;
1445 u64 listen_vpp_handle;
1448 listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1449 if (!listen_session || (listen_session->flags & VCL_SESSION_F_IS_VEP))
1450 return VPPCOM_EBADFD;
1452 if (q_len == 0 || q_len == ~0)
1453 q_len = vcm->cfg.listen_queue_size;
1455 listen_vpp_handle = listen_session->vpp_handle;
1456 if (listen_session->session_state == VCL_STATE_LISTEN)
1458 VDBG (0, "session %u [0x%llx]: already in listen state!",
1459 listen_sh, listen_vpp_handle);
1463 VDBG (0, "session %u: sending vpp listen request...", listen_sh);
1466 * Send listen request to vpp and wait for reply
1468 vcl_send_session_listen (wrk, listen_session);
1469 rv = vppcom_wait_for_session_state_change (listen_session->session_index,
1471 vcm->cfg.session_timeout);
1473 if (PREDICT_FALSE (rv))
1475 listen_session = vcl_session_get_w_handle (wrk, listen_sh);
1476 VDBG (0, "session %u [0x%llx]: listen failed! returning %d (%s)",
1477 listen_sh, listen_session->vpp_handle, rv,
1478 vppcom_retval_str (rv));
1486 validate_args_session_accept_ (vcl_worker_t * wrk, vcl_session_t * ls)
1488 if (ls->flags & VCL_SESSION_F_IS_VEP)
1490 VDBG (0, "ERROR: cannot accept on epoll session %u!",
1492 return VPPCOM_EBADFD;
1495 if ((ls->session_state != VCL_STATE_LISTEN)
1496 && (!vcl_session_is_connectable_listener (wrk, ls)))
1498 VDBG (0, "ERROR: session [0x%llx]: not in listen state! state 0x%x"
1499 " (%s)", ls->vpp_handle, ls->session_state,
1500 vppcom_session_state_str (ls->session_state));
1501 return VPPCOM_EBADFD;
1507 vppcom_unformat_proto (uint8_t * proto, char *proto_str)
1509 if (!strcmp (proto_str, "TCP"))
1510 *proto = VPPCOM_PROTO_TCP;
1511 else if (!strcmp (proto_str, "tcp"))
1512 *proto = VPPCOM_PROTO_TCP;
1513 else if (!strcmp (proto_str, "UDP"))
1514 *proto = VPPCOM_PROTO_UDP;
1515 else if (!strcmp (proto_str, "udp"))
1516 *proto = VPPCOM_PROTO_UDP;
1517 else if (!strcmp (proto_str, "TLS"))
1518 *proto = VPPCOM_PROTO_TLS;
1519 else if (!strcmp (proto_str, "tls"))
1520 *proto = VPPCOM_PROTO_TLS;
1521 else if (!strcmp (proto_str, "QUIC"))
1522 *proto = VPPCOM_PROTO_QUIC;
1523 else if (!strcmp (proto_str, "quic"))
1524 *proto = VPPCOM_PROTO_QUIC;
1531 vppcom_session_accept (uint32_t listen_session_handle, vppcom_endpt_t * ep,
1534 u32 client_session_index = ~0, listen_session_index, accept_flags = 0;
1535 vcl_worker_t *wrk = vcl_worker_get_current ();
1536 session_accepted_msg_t accepted_msg;
1537 vcl_session_t *listen_session = 0;
1538 vcl_session_t *client_session = 0;
1539 vcl_session_msg_t *evt;
1540 svm_msg_q_msg_t msg;
1545 listen_session = vcl_session_get_w_handle (wrk, listen_session_handle);
1546 if (!listen_session)
1547 return VPPCOM_EBADFD;
1549 listen_session_index = listen_session->session_index;
1550 if ((rv = validate_args_session_accept_ (wrk, listen_session)))
1553 if (clib_fifo_elts (listen_session->accept_evts_fifo))
1555 clib_fifo_sub2 (listen_session->accept_evts_fifo, evt);
1556 accept_flags = evt->flags;
1557 accepted_msg = evt->accepted_msg;
1561 is_nonblocking = vcl_session_has_attr (listen_session,
1562 VCL_SESS_ATTR_NONBLOCK);
1565 if (svm_msg_q_is_empty (wrk->app_event_queue) && is_nonblocking)
1566 return VPPCOM_EAGAIN;
1568 if (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_WAIT, 0))
1569 return VPPCOM_EAGAIN;
1571 e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
1572 if (e->event_type != SESSION_CTRL_EVT_ACCEPTED)
1574 vcl_handle_mq_event (wrk, e);
1575 svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1578 clib_memcpy_fast (&accepted_msg, e->data, sizeof (accepted_msg));
1579 svm_msg_q_free_msg (wrk->app_event_queue, &msg);
1585 client_session_index = vcl_session_accepted_handler (wrk, &accepted_msg,
1586 listen_session_index);
1587 if (client_session_index == VCL_INVALID_SESSION_INDEX)
1588 return VPPCOM_ECONNABORTED;
1590 listen_session = vcl_session_get (wrk, listen_session_index);
1591 client_session = vcl_session_get (wrk, client_session_index);
1593 if (flags & O_NONBLOCK)
1594 vcl_session_set_attr (client_session, VCL_SESS_ATTR_NONBLOCK);
1596 VDBG (1, "listener %u [0x%llx]: Got a connect request! session %u [0x%llx],"
1597 " flags %d, is_nonblocking %u", listen_session->session_index,
1598 listen_session->vpp_handle, client_session_index,
1599 client_session->vpp_handle, flags,
1600 vcl_session_has_attr (client_session, VCL_SESS_ATTR_NONBLOCK));
1604 ep->is_ip4 = client_session->transport.is_ip4;
1605 ep->port = client_session->transport.rmt_port;
1606 if (client_session->transport.is_ip4)
1607 clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip4,
1608 sizeof (ip4_address_t));
1610 clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip6,
1611 sizeof (ip6_address_t));
1614 VDBG (0, "listener %u [0x%llx] accepted %u [0x%llx] peer: %U:%u "
1615 "local: %U:%u", listen_session_handle, listen_session->vpp_handle,
1616 client_session_index, client_session->vpp_handle,
1617 format_ip46_address, &client_session->transport.rmt_ip,
1618 client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1619 clib_net_to_host_u16 (client_session->transport.rmt_port),
1620 format_ip46_address, &client_session->transport.lcl_ip,
1621 client_session->transport.is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
1622 clib_net_to_host_u16 (client_session->transport.lcl_port));
1623 vcl_evt (VCL_EVT_ACCEPT, client_session, listen_session,
1624 client_session_index);
1627 * Session might have been closed already
1631 if (accept_flags & VCL_ACCEPTED_F_CLOSED)
1632 client_session->session_state = VCL_STATE_VPP_CLOSING;
1633 else if (accept_flags & VCL_ACCEPTED_F_RESET)
1634 client_session->session_state = VCL_STATE_DISCONNECT;
1636 return vcl_session_handle (client_session);
1640 vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep)
1642 vcl_worker_t *wrk = vcl_worker_get_current ();
1643 vcl_session_t *session = 0;
1647 session = vcl_session_get_w_handle (wrk, session_handle);
1649 return VPPCOM_EBADFD;
1650 session_index = session->session_index;
1652 if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1654 VDBG (0, "ERROR: cannot connect epoll session %u!",
1655 session->session_index);
1656 return VPPCOM_EBADFD;
1659 if (PREDICT_FALSE (vcl_session_is_ready (session)))
1661 VDBG (0, "session handle %u [0x%llx]: session already "
1662 "connected to %s %U port %d proto %s, state 0x%x (%s)",
1663 session_handle, session->vpp_handle,
1664 session->transport.is_ip4 ? "IPv4" : "IPv6", format_ip46_address,
1665 &session->transport.rmt_ip, session->transport.is_ip4 ?
1666 IP46_TYPE_IP4 : IP46_TYPE_IP6,
1667 clib_net_to_host_u16 (session->transport.rmt_port),
1668 vppcom_proto_str (session->session_type), session->session_state,
1669 vppcom_session_state_str (session->session_state));
1673 /* Attempt to connect a connectionless listener */
1674 if (PREDICT_FALSE (session->session_state == VCL_STATE_LISTEN))
1676 if (session->session_type != VPPCOM_PROTO_UDP)
1677 return VPPCOM_EINVAL;
1678 vcl_send_session_unlisten (wrk, session);
1679 session->session_state = VCL_STATE_CLOSED;
1682 session->transport.is_ip4 = server_ep->is_ip4;
1683 vcl_ip_copy_from_ep (&session->transport.rmt_ip, server_ep);
1684 session->transport.rmt_port = server_ep->port;
1685 session->parent_handle = VCL_INVALID_SESSION_HANDLE;
1686 session->flags |= VCL_SESSION_F_CONNECTED;
1688 VDBG (0, "session handle %u (%s): connecting to peer %s %U "
1689 "port %d proto %s", session_handle,
1690 vppcom_session_state_str (session->session_state),
1691 session->transport.is_ip4 ? "IPv4" : "IPv6",
1692 format_ip46_address,
1693 &session->transport.rmt_ip, session->transport.is_ip4 ?
1694 IP46_TYPE_IP4 : IP46_TYPE_IP6,
1695 clib_net_to_host_u16 (session->transport.rmt_port),
1696 vppcom_proto_str (session->session_type));
1698 vcl_send_session_connect (wrk, session);
1700 if (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK))
1702 /* State set to STATE_UPDATED to ensure the session is not assumed
1703 * to be ready and to also allow the app to close it prior to vpp's
1704 * connected reply. */
1705 session->session_state = VCL_STATE_UPDATED;
1706 return VPPCOM_EINPROGRESS;
1710 * Wait for reply from vpp if blocking
1712 rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1713 vcm->cfg.session_timeout);
1715 session = vcl_session_get (wrk, session_index);
1716 VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1717 session->vpp_handle, rv ? "failed" : "succeeded");
1723 vppcom_session_stream_connect (uint32_t session_handle,
1724 uint32_t parent_session_handle)
1726 vcl_worker_t *wrk = vcl_worker_get_current ();
1727 vcl_session_t *session, *parent_session;
1728 u32 session_index, parent_session_index;
1731 session = vcl_session_get_w_handle (wrk, session_handle);
1733 return VPPCOM_EBADFD;
1734 parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1735 if (!parent_session)
1736 return VPPCOM_EBADFD;
1738 session_index = session->session_index;
1739 parent_session_index = parent_session->session_index;
1740 if (PREDICT_FALSE (session->flags & VCL_SESSION_F_IS_VEP))
1742 VDBG (0, "ERROR: cannot connect epoll session %u!",
1743 session->session_index);
1744 return VPPCOM_EBADFD;
1747 if (PREDICT_FALSE (vcl_session_is_ready (session)))
1749 VDBG (0, "session handle %u [0x%llx]: session already "
1750 "connected to session %u [0x%llx] proto %s, state 0x%x (%s)",
1751 session_handle, session->vpp_handle,
1752 parent_session_handle, parent_session->vpp_handle,
1753 vppcom_proto_str (session->session_type), session->session_state,
1754 vppcom_session_state_str (session->session_state));
1758 /* Connect to quic session specifics */
1759 session->transport.is_ip4 = parent_session->transport.is_ip4;
1760 session->transport.rmt_ip.ip4.as_u32 = (uint32_t) 1;
1761 session->transport.rmt_port = 0;
1762 session->parent_handle = parent_session->vpp_handle;
1764 VDBG (0, "session handle %u: connecting to session %u [0x%llx]",
1765 session_handle, parent_session_handle, parent_session->vpp_handle);
1768 * Send connect request and wait for reply from vpp
1770 vcl_send_session_connect (wrk, session);
1771 rv = vppcom_wait_for_session_state_change (session_index, VCL_STATE_READY,
1772 vcm->cfg.session_timeout);
1774 session->listener_index = parent_session_index;
1775 parent_session = vcl_session_get_w_handle (wrk, parent_session_handle);
1777 parent_session->n_accepted_sessions++;
1779 session = vcl_session_get (wrk, session_index);
1780 VDBG (0, "session %u [0x%llx]: connect %s!", session->session_index,
1781 session->vpp_handle, rv ? "failed" : "succeeded");
1787 vcl_is_rx_evt_for_session (session_event_t * e, u32 sid, u8 is_ct)
1789 return (e->event_type == SESSION_IO_EVT_RX && e->session_index == sid);
1793 vppcom_session_read_internal (uint32_t session_handle, void *buf, int n,
1796 vcl_worker_t *wrk = vcl_worker_get_current ();
1797 int rv, n_read = 0, is_nonblocking;
1798 vcl_session_t *s = 0;
1799 svm_fifo_t *rx_fifo;
1800 svm_msg_q_msg_t msg;
1805 if (PREDICT_FALSE (!buf))
1806 return VPPCOM_EINVAL;
1808 s = vcl_session_get_w_handle (wrk, session_handle);
1809 if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
1810 return VPPCOM_EBADFD;
1812 if (PREDICT_FALSE (!vcl_session_is_open (s)))
1814 VDBG (0, "session %u[0x%llx] is not open! state 0x%x (%s)",
1815 s->session_index, s->vpp_handle, s->session_state,
1816 vppcom_session_state_str (s->session_state));
1817 return vcl_session_closed_error (s);
1820 is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
1821 is_ct = vcl_session_is_ct (s);
1822 mq = wrk->app_event_queue;
1823 rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
1824 s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
1826 if (svm_fifo_is_empty_cons (rx_fifo))
1830 if (vcl_session_is_closing (s))
1831 return vcl_session_closing_error (s);
1833 svm_fifo_unset_event (s->rx_fifo);
1834 svm_fifo_unset_event (rx_fifo);
1835 return VPPCOM_EWOULDBLOCK;
1837 while (svm_fifo_is_empty_cons (rx_fifo))
1839 if (vcl_session_is_closing (s))
1840 return vcl_session_closing_error (s);
1843 svm_fifo_unset_event (s->rx_fifo);
1844 svm_fifo_unset_event (rx_fifo);
1845 svm_msg_q_lock (mq);
1846 if (svm_msg_q_is_empty (mq))
1847 svm_msg_q_wait (mq);
1849 svm_msg_q_sub_w_lock (mq, &msg);
1850 e = svm_msg_q_msg_data (mq, &msg);
1851 svm_msg_q_unlock (mq);
1852 if (!vcl_is_rx_evt_for_session (e, s->session_index, is_ct))
1853 vcl_handle_mq_event (wrk, e);
1854 svm_msg_q_free_msg (mq, &msg);
1861 rv = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 0, peek);
1863 rv = app_recv_stream_raw (rx_fifo, buf, n, 0, peek);
1868 if (svm_fifo_is_empty_cons (rx_fifo))
1871 svm_fifo_unset_event (s->rx_fifo);
1872 svm_fifo_unset_event (rx_fifo);
1873 if (!svm_fifo_is_empty_cons (rx_fifo)
1874 && svm_fifo_set_event (rx_fifo) && is_nonblocking)
1876 vec_add2 (wrk->unhandled_evts_vector, e, 1);
1877 e->event_type = SESSION_IO_EVT_RX;
1878 e->session_index = s->session_index;
1881 else if (PREDICT_FALSE (rv < n && !s->is_dgram))
1883 /* More data enqueued while reading. Try to drain it
1884 * or fill the buffer. Avoid doing that for dgrams */
1890 if (PREDICT_FALSE (svm_fifo_needs_deq_ntf (rx_fifo, n_read)))
1892 svm_fifo_clear_deq_ntf (rx_fifo);
1893 app_send_io_evt_to_vpp (s->vpp_evt_q,
1894 s->rx_fifo->shr->master_session_index,
1895 SESSION_IO_EVT_RX, SVM_Q_WAIT);
1898 VDBG (2, "session %u[0x%llx]: read %d bytes from (%p)", s->session_index,
1899 s->vpp_handle, n_read, rx_fifo);
1905 vppcom_session_read (uint32_t session_handle, void *buf, size_t n)
1907 return (vppcom_session_read_internal (session_handle, buf, n, 0));
1911 vppcom_session_peek (uint32_t session_handle, void *buf, int n)
1913 return (vppcom_session_read_internal (session_handle, buf, n, 1));
1917 vppcom_session_read_segments (uint32_t session_handle,
1918 vppcom_data_segment_t * ds, uint32_t n_segments,
1921 vcl_worker_t *wrk = vcl_worker_get_current ();
1922 int n_read = 0, is_nonblocking;
1923 vcl_session_t *s = 0;
1924 svm_fifo_t *rx_fifo;
1925 svm_msg_q_msg_t msg;
1930 s = vcl_session_get_w_handle (wrk, session_handle);
1931 if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
1932 return VPPCOM_EBADFD;
1934 if (PREDICT_FALSE (!vcl_session_is_open (s)))
1935 return vcl_session_closed_error (s);
1937 is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
1938 is_ct = vcl_session_is_ct (s);
1939 mq = wrk->app_event_queue;
1940 rx_fifo = is_ct ? s->ct_rx_fifo : s->rx_fifo;
1941 s->flags &= ~VCL_SESSION_F_HAS_RX_EVT;
1943 if (svm_fifo_is_empty_cons (rx_fifo))
1948 svm_fifo_unset_event (s->rx_fifo);
1949 svm_fifo_unset_event (rx_fifo);
1950 return VPPCOM_EWOULDBLOCK;
1952 while (svm_fifo_is_empty_cons (rx_fifo))
1954 if (vcl_session_is_closing (s))
1955 return vcl_session_closing_error (s);
1958 svm_fifo_unset_event (s->rx_fifo);
1959 svm_fifo_unset_event (rx_fifo);
1960 svm_msg_q_lock (mq);
1961 if (svm_msg_q_is_empty (mq))
1962 svm_msg_q_wait (mq);
1964 svm_msg_q_sub_w_lock (mq, &msg);
1965 e = svm_msg_q_msg_data (mq, &msg);
1966 svm_msg_q_unlock (mq);
1967 if (!vcl_is_rx_evt_for_session (e, s->session_index, is_ct))
1968 vcl_handle_mq_event (wrk, e);
1969 svm_msg_q_free_msg (mq, &msg);
1973 n_read = svm_fifo_segments (rx_fifo, s->rx_bytes_pending,
1974 (svm_fifo_seg_t *) ds, n_segments, max_bytes);
1976 return VPPCOM_EAGAIN;
1978 if (svm_fifo_max_dequeue_cons (rx_fifo) == n_read)
1981 svm_fifo_unset_event (s->rx_fifo);
1982 svm_fifo_unset_event (rx_fifo);
1983 if (svm_fifo_max_dequeue_cons (rx_fifo) != n_read
1984 && svm_fifo_set_event (rx_fifo)
1985 && vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK))
1988 vec_add2 (wrk->unhandled_evts_vector, e, 1);
1989 e->event_type = SESSION_IO_EVT_RX;
1990 e->session_index = s->session_index;
1994 s->rx_bytes_pending += n_read;
1999 vppcom_session_free_segments (uint32_t session_handle, uint32_t n_bytes)
2001 vcl_worker_t *wrk = vcl_worker_get_current ();
2005 s = vcl_session_get_w_handle (wrk, session_handle);
2006 if (PREDICT_FALSE (!s || (s->flags & VCL_SESSION_F_IS_VEP)))
2009 is_ct = vcl_session_is_ct (s);
2010 svm_fifo_dequeue_drop (is_ct ? s->ct_rx_fifo : s->rx_fifo, n_bytes);
2012 ASSERT (s->rx_bytes_pending < n_bytes);
2013 s->rx_bytes_pending -= n_bytes;
2017 vcl_is_tx_evt_for_session (session_event_t * e, u32 sid, u8 is_ct)
2019 return (e->event_type == SESSION_IO_EVT_TX && e->session_index == sid);
2023 vcl_fifo_is_writeable (svm_fifo_t * f, u32 len, u8 is_dgram)
2025 u32 max_enq = svm_fifo_max_enqueue_prod (f);
2027 return max_enq >= (sizeof (session_dgram_hdr_t) + len);
2033 vppcom_session_write_inline (vcl_worker_t * wrk, vcl_session_t * s, void *buf,
2034 size_t n, u8 is_flush, u8 is_dgram)
2036 int n_write, is_nonblocking;
2037 session_evt_type_t et;
2038 svm_msg_q_msg_t msg;
2039 svm_fifo_t *tx_fifo;
2044 if (PREDICT_FALSE (!buf || n == 0))
2045 return VPPCOM_EINVAL;
2047 if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2049 VDBG (0, "ERROR: session %u [0x%llx]: cannot write to an epoll"
2050 " session!", s->session_index, s->vpp_handle);
2051 return VPPCOM_EBADFD;
2054 if (PREDICT_FALSE (!vcl_session_is_open (s)))
2056 VDBG (1, "session %u [0x%llx]: is not open! state 0x%x (%s)",
2057 s->session_index, s->vpp_handle, s->session_state,
2058 vppcom_session_state_str (s->session_state));
2059 return vcl_session_closed_error (s);;
2062 is_ct = vcl_session_is_ct (s);
2063 tx_fifo = is_ct ? s->ct_tx_fifo : s->tx_fifo;
2064 is_nonblocking = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
2066 mq = wrk->app_event_queue;
2067 if (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2071 return VPPCOM_EWOULDBLOCK;
2073 while (!vcl_fifo_is_writeable (tx_fifo, n, is_dgram))
2075 svm_fifo_add_want_deq_ntf (tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2076 if (vcl_session_is_closing (s))
2077 return vcl_session_closing_error (s);
2078 svm_msg_q_lock (mq);
2079 if (svm_msg_q_is_empty (mq))
2080 svm_msg_q_wait (mq);
2082 svm_msg_q_sub_w_lock (mq, &msg);
2083 e = svm_msg_q_msg_data (mq, &msg);
2084 svm_msg_q_unlock (mq);
2086 if (!vcl_is_tx_evt_for_session (e, s->session_index, is_ct))
2087 vcl_handle_mq_event (wrk, e);
2088 svm_msg_q_free_msg (mq, &msg);
2092 et = SESSION_IO_EVT_TX;
2093 if (is_flush && !is_ct)
2094 et = SESSION_IO_EVT_TX_FLUSH;
2097 n_write = app_send_dgram_raw (tx_fifo, &s->transport,
2098 s->vpp_evt_q, buf, n, et,
2099 0 /* do_evt */ , SVM_Q_WAIT);
2101 n_write = app_send_stream_raw (tx_fifo, s->vpp_evt_q, buf, n, et,
2102 0 /* do_evt */ , SVM_Q_WAIT);
2104 if (svm_fifo_set_event (s->tx_fifo))
2105 app_send_io_evt_to_vpp (
2106 s->vpp_evt_q, s->tx_fifo->shr->master_session_index, et, SVM_Q_WAIT);
2108 /* The underlying fifo segment can run out of memory */
2109 if (PREDICT_FALSE (n_write < 0))
2110 return VPPCOM_EAGAIN;
2112 VDBG (2, "session %u [0x%llx]: wrote %d bytes", s->session_index,
2113 s->vpp_handle, n_write);
2119 vppcom_session_write (uint32_t session_handle, void *buf, size_t n)
2121 vcl_worker_t *wrk = vcl_worker_get_current ();
2124 s = vcl_session_get_w_handle (wrk, session_handle);
2125 if (PREDICT_FALSE (!s))
2126 return VPPCOM_EBADFD;
2128 return vppcom_session_write_inline (wrk, s, buf, n,
2129 0 /* is_flush */ , s->is_dgram ? 1 : 0);
2133 vppcom_session_write_msg (uint32_t session_handle, void *buf, size_t n)
2135 vcl_worker_t *wrk = vcl_worker_get_current ();
2138 s = vcl_session_get_w_handle (wrk, session_handle);
2139 if (PREDICT_FALSE (!s))
2140 return VPPCOM_EBADFD;
2142 return vppcom_session_write_inline (wrk, s, buf, n,
2143 1 /* is_flush */ , s->is_dgram ? 1 : 0);
2146 #define vcl_fifo_rx_evt_valid_or_break(_s) \
2147 if (PREDICT_FALSE (!_s->rx_fifo)) \
2149 if (PREDICT_FALSE (svm_fifo_is_empty (_s->rx_fifo))) \
2151 if (!vcl_session_is_ct (_s)) \
2153 svm_fifo_unset_event (_s->rx_fifo); \
2154 if (svm_fifo_is_empty (_s->rx_fifo)) \
2157 else if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2159 svm_fifo_unset_event (_s->rx_fifo); /* rx evts on actual fifo*/ \
2160 if (svm_fifo_is_empty (_s->ct_rx_fifo)) \
2166 vcl_select_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2167 unsigned long n_bits, unsigned long *read_map,
2168 unsigned long *write_map,
2169 unsigned long *except_map, u32 * bits_set)
2171 session_disconnected_msg_t *disconnected_msg;
2172 session_connected_msg_t *connected_msg;
2176 switch (e->event_type)
2178 case SESSION_IO_EVT_RX:
2179 sid = e->session_index;
2180 s = vcl_session_get (wrk, sid);
2181 if (!s || !vcl_session_is_open (s))
2183 vcl_fifo_rx_evt_valid_or_break (s);
2184 if (sid < n_bits && read_map)
2186 clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2190 case SESSION_IO_EVT_TX:
2191 sid = e->session_index;
2192 s = vcl_session_get (wrk, sid);
2193 if (!s || !vcl_session_is_open (s))
2195 if (sid < n_bits && write_map)
2197 clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2201 case SESSION_CTRL_EVT_ACCEPTED:
2203 s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
2205 s = vcl_session_get (wrk, e->session_index);
2208 sid = s->session_index;
2209 if (sid < n_bits && read_map)
2211 clib_bitmap_set_no_check ((uword *) read_map, sid, 1);
2215 case SESSION_CTRL_EVT_CONNECTED:
2218 connected_msg = (session_connected_msg_t *) e->data;
2219 sid = vcl_session_connected_handler (wrk, connected_msg);
2222 sid = e->session_index;
2223 if (sid == VCL_INVALID_SESSION_INDEX)
2225 if (sid < n_bits && write_map)
2227 clib_bitmap_set_no_check ((uword *) write_map, sid, 1);
2231 case SESSION_CTRL_EVT_DISCONNECTED:
2232 disconnected_msg = (session_disconnected_msg_t *) e->data;
2233 s = vcl_session_disconnected_handler (wrk, disconnected_msg);
2236 sid = s->session_index;
2237 if (sid < n_bits && except_map)
2239 clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2243 case SESSION_CTRL_EVT_RESET:
2244 sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2245 if (sid < n_bits && except_map)
2247 clib_bitmap_set_no_check ((uword *) except_map, sid, 1);
2251 case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2252 vcl_session_unlisten_reply_handler (wrk, e->data);
2254 case SESSION_CTRL_EVT_MIGRATED:
2255 vcl_session_migrated_handler (wrk, e->data);
2257 case SESSION_CTRL_EVT_CLEANUP:
2258 vcl_session_cleanup_handler (wrk, e->data);
2260 case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2261 vcl_session_worker_update_reply_handler (wrk, e->data);
2263 case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2264 vcl_session_req_worker_update_handler (wrk, e->data);
2266 case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2267 vcl_session_app_add_segment_handler (wrk, e->data);
2269 case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2270 vcl_session_app_del_segment_handler (wrk, e->data);
2272 case SESSION_CTRL_EVT_APP_WRK_RPC:
2273 vcl_worker_rpc_handler (wrk, e->data);
2276 clib_warning ("unhandled: %u", e->event_type);
2282 vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2283 unsigned long n_bits, unsigned long *read_map,
2284 unsigned long *write_map, unsigned long *except_map,
2285 double time_to_wait, u32 * bits_set)
2287 svm_msg_q_msg_t *msg;
2291 svm_msg_q_lock (mq);
2292 if (svm_msg_q_is_empty (mq))
2296 svm_msg_q_unlock (mq);
2302 svm_msg_q_unlock (mq);
2305 else if (time_to_wait < 0)
2307 svm_msg_q_wait (mq);
2311 if (svm_msg_q_timedwait (mq, time_to_wait))
2313 svm_msg_q_unlock (mq);
2318 vcl_mq_dequeue_batch (wrk, mq, ~0);
2319 svm_msg_q_unlock (mq);
2321 for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2323 msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2324 e = svm_msg_q_msg_data (mq, msg);
2325 vcl_select_handle_mq_event (wrk, e, n_bits, read_map, write_map,
2326 except_map, bits_set);
2327 svm_msg_q_free_msg (mq, msg);
2329 vec_reset_length (wrk->mq_msg_vector);
2330 vcl_handle_pending_wrk_updates (wrk);
2335 vppcom_select_condvar (vcl_worker_t * wrk, int n_bits,
2336 vcl_si_set * read_map, vcl_si_set * write_map,
2337 vcl_si_set * except_map, double time_to_wait,
2340 double wait = 0, start = 0;
2344 wait = time_to_wait;
2345 start = clib_time_now (&wrk->clib_time);
2350 vcl_select_handle_mq (wrk, wrk->app_event_queue, n_bits, read_map,
2351 write_map, except_map, wait, bits_set);
2357 wait = wait - (clib_time_now (&wrk->clib_time) - start);
2365 vppcom_select_eventfd (vcl_worker_t * wrk, int n_bits,
2366 vcl_si_set * read_map, vcl_si_set * write_map,
2367 vcl_si_set * except_map, double time_to_wait,
2370 vcl_mq_evt_conn_t *mqc;
2371 int __clib_unused n_read;
2375 vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
2376 n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
2377 vec_len (wrk->mq_events), time_to_wait);
2378 for (i = 0; i < n_mq_evts; i++)
2380 mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
2381 n_read = read (mqc->mq_fd, &buf, sizeof (buf));
2382 vcl_select_handle_mq (wrk, mqc->mq, n_bits, read_map, write_map,
2383 except_map, 0, bits_set);
2386 return (n_mq_evts > 0 ? (int) *bits_set : 0);
2390 vppcom_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
2391 vcl_si_set * except_map, double time_to_wait)
2393 u32 sid, minbits = clib_max (n_bits, BITS (uword)), bits_set = 0;
2394 vcl_worker_t *wrk = vcl_worker_get_current ();
2395 vcl_session_t *session = 0;
2398 if (n_bits && read_map)
2400 clib_bitmap_validate (wrk->rd_bitmap, minbits);
2401 clib_memcpy_fast (wrk->rd_bitmap, read_map,
2402 vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2403 memset (read_map, 0, vec_len (wrk->rd_bitmap) * sizeof (vcl_si_set));
2405 if (n_bits && write_map)
2407 clib_bitmap_validate (wrk->wr_bitmap, minbits);
2408 clib_memcpy_fast (wrk->wr_bitmap, write_map,
2409 vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2410 memset (write_map, 0, vec_len (wrk->wr_bitmap) * sizeof (vcl_si_set));
2412 if (n_bits && except_map)
2414 clib_bitmap_validate (wrk->ex_bitmap, minbits);
2415 clib_memcpy_fast (wrk->ex_bitmap, except_map,
2416 vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2417 memset (except_map, 0, vec_len (wrk->ex_bitmap) * sizeof (vcl_si_set));
2427 clib_bitmap_foreach (sid, wrk->wr_bitmap) {
2428 if (!(session = vcl_session_get (wrk, sid)))
2430 clib_bitmap_set_no_check ((uword*)write_map, sid, 1);
2435 if (vcl_session_write_ready (session))
2437 clib_bitmap_set_no_check ((uword*)write_map, sid, 1);
2441 svm_fifo_add_want_deq_ntf (session->tx_fifo, SVM_FIFO_WANT_DEQ_NOTIF);
2448 clib_bitmap_foreach (sid, wrk->rd_bitmap) {
2449 if (!(session = vcl_session_get (wrk, sid)))
2451 clib_bitmap_set_no_check ((uword*)read_map, sid, 1);
2456 if (vcl_session_read_ready (session))
2458 clib_bitmap_set_no_check ((uword*)read_map, sid, 1);
2466 for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
2468 vcl_select_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i], n_bits,
2469 read_map, write_map, except_map, &bits_set);
2471 vec_reset_length (wrk->unhandled_evts_vector);
2473 if (vcm->cfg.use_mq_eventfd)
2474 vppcom_select_eventfd (wrk, n_bits, read_map, write_map, except_map,
2475 time_to_wait, &bits_set);
2477 vppcom_select_condvar (wrk, n_bits, read_map, write_map, except_map,
2478 time_to_wait, &bits_set);
2484 vep_verify_epoll_chain (vcl_worker_t * wrk, u32 vep_handle)
2486 vppcom_epoll_t *vep;
2487 u32 sh = vep_handle;
2490 if (VPPCOM_DEBUG <= 2)
2493 s = vcl_session_get_w_handle (wrk, vep_handle);
2494 if (PREDICT_FALSE (!s))
2496 VDBG (0, "ERROR: Invalid vep_sh (%u)!", vep_handle);
2499 if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP)))
2501 VDBG (0, "ERROR: vep_sh (%u) is not a vep!", vep_handle);
2505 VDBG (0, "vep_sh (%u): Dumping epoll chain\n"
2508 " is_vep_session = %u\n"
2509 " next_sh = 0x%x (%u)\n"
2510 "}\n", vep_handle, s->flags & VCL_SESSION_F_IS_VEP,
2511 s->flags & VCL_SESSION_F_IS_VEP_SESSION, vep->next_sh, vep->next_sh);
2513 for (sh = vep->next_sh; sh != ~0; sh = vep->next_sh)
2515 s = vcl_session_get_w_handle (wrk, sh);
2516 if (PREDICT_FALSE (!s))
2518 VDBG (0, "ERROR: Invalid sh (%u)!", sh);
2521 if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2523 VDBG (0, "ERROR: sh (%u) is a vep!", vep_handle);
2525 else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2527 VDBG (0, "ERROR: sh (%u) is not a vep session handle!", sh);
2531 if (PREDICT_FALSE (vep->vep_sh != vep_handle))
2532 VDBG (0, "ERROR: session (%u) vep_sh (%u) != vep_sh (%u)!",
2533 sh, s->vep.vep_sh, vep_handle);
2534 if (s->flags & VCL_SESSION_F_IS_VEP_SESSION)
2536 VDBG (0, "vep_sh[%u]: sh 0x%x (%u)\n"
2538 " next_sh = 0x%x (%u)\n"
2539 " prev_sh = 0x%x (%u)\n"
2540 " vep_sh = 0x%x (%u)\n"
2541 " ev.events = 0x%x\n"
2542 " ev.data.u64 = 0x%llx\n"
2545 vep_handle, sh, sh, vep->next_sh, vep->next_sh, vep->prev_sh,
2546 vep->prev_sh, vep->vep_sh, vep->vep_sh, vep->ev.events,
2547 vep->ev.data.u64, vep->et_mask);
2552 VDBG (0, "vep_sh (%u): Dump complete!\n", vep_handle);
2556 vppcom_epoll_create (void)
2558 vcl_worker_t *wrk = vcl_worker_get_current ();
2559 vcl_session_t *vep_session;
2561 vep_session = vcl_session_alloc (wrk);
2563 vep_session->flags |= VCL_SESSION_F_IS_VEP;
2564 vep_session->vep.vep_sh = ~0;
2565 vep_session->vep.next_sh = ~0;
2566 vep_session->vep.prev_sh = ~0;
2567 vep_session->vpp_handle = ~0;
2569 vcl_evt (VCL_EVT_EPOLL_CREATE, vep_session, vep_session->session_index);
2570 VDBG (0, "Created vep_idx %u", vep_session->session_index);
2572 return vcl_session_handle (vep_session);
2576 vppcom_epoll_ctl (uint32_t vep_handle, int op, uint32_t session_handle,
2577 struct epoll_event *event)
2579 vcl_worker_t *wrk = vcl_worker_get_current ();
2580 vcl_session_t *vep_session;
2585 if (vep_handle == session_handle)
2587 VDBG (0, "vep_sh == session handle (%u)!", vep_handle);
2588 return VPPCOM_EINVAL;
2591 vep_session = vcl_session_get_w_handle (wrk, vep_handle);
2592 if (PREDICT_FALSE (!vep_session))
2594 VDBG (0, "Invalid vep_sh (%u)!", vep_handle);
2595 return VPPCOM_EBADFD;
2597 if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
2599 VDBG (0, "vep_sh (%u) is not a vep!", vep_handle);
2600 return VPPCOM_EINVAL;
2603 ASSERT (vep_session->vep.vep_sh == ~0);
2604 ASSERT (vep_session->vep.prev_sh == ~0);
2606 s = vcl_session_get_w_handle (wrk, session_handle);
2607 if (PREDICT_FALSE (!s))
2609 VDBG (0, "Invalid session_handle (%u)!", session_handle);
2610 return VPPCOM_EBADFD;
2612 if (PREDICT_FALSE (s->flags & VCL_SESSION_F_IS_VEP))
2614 VDBG (0, "session_handle (%u) is a vep!", vep_handle);
2615 return VPPCOM_EINVAL;
2621 if (PREDICT_FALSE (!event))
2623 VDBG (0, "EPOLL_CTL_ADD: NULL pointer to epoll_event structure!");
2624 return VPPCOM_EINVAL;
2626 if (vep_session->vep.next_sh != ~0)
2628 vcl_session_t *next_session;
2629 next_session = vcl_session_get_w_handle (wrk,
2630 vep_session->vep.next_sh);
2631 if (PREDICT_FALSE (!next_session))
2633 VDBG (0, "EPOLL_CTL_ADD: Invalid vep.next_sh (%u) on "
2634 "vep_idx (%u)!", vep_session->vep.next_sh, vep_handle);
2635 return VPPCOM_EBADFD;
2637 ASSERT (next_session->vep.prev_sh == vep_handle);
2638 next_session->vep.prev_sh = session_handle;
2640 s->vep.next_sh = vep_session->vep.next_sh;
2641 s->vep.prev_sh = vep_handle;
2642 s->vep.vep_sh = vep_handle;
2643 s->vep.et_mask = VEP_DEFAULT_ET_MASK;
2645 s->flags &= ~VCL_SESSION_F_IS_VEP;
2646 s->flags |= VCL_SESSION_F_IS_VEP_SESSION;
2647 vep_session->vep.next_sh = session_handle;
2649 txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
2650 if (txf && (event->events & EPOLLOUT))
2651 svm_fifo_add_want_deq_ntf (txf, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2653 /* Generate EPOLLOUT if tx fifo not full */
2654 if ((event->events & EPOLLOUT) && (vcl_session_write_ready (s) > 0))
2656 session_event_t e = { 0 };
2657 e.event_type = SESSION_IO_EVT_TX;
2658 e.session_index = s->session_index;
2659 vec_add1 (wrk->unhandled_evts_vector, e);
2661 /* Generate EPOLLIN if rx fifo has data */
2662 if ((event->events & EPOLLIN) && (vcl_session_read_ready (s) > 0))
2664 session_event_t e = { 0 };
2665 e.event_type = SESSION_IO_EVT_RX;
2666 e.session_index = s->session_index;
2667 vec_add1 (wrk->unhandled_evts_vector, e);
2669 VDBG (1, "EPOLL_CTL_ADD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2670 vep_handle, session_handle, event->events, event->data.u64);
2671 vcl_evt (VCL_EVT_EPOLL_CTLADD, s, event->events, event->data.u64);
2675 if (PREDICT_FALSE (!event))
2677 VDBG (0, "EPOLL_CTL_MOD: NULL pointer to epoll_event structure!");
2681 else if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2683 VDBG (0, "sh %u EPOLL_CTL_MOD: not a vep session!", session_handle);
2687 else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
2689 VDBG (0, "EPOLL_CTL_MOD: sh %u vep_sh (%u) != vep_sh (%u)!",
2690 session_handle, s->vep.vep_sh, vep_handle);
2695 /* Generate EPOLLOUT when tx_fifo/ct_tx_fifo not full */
2696 if ((event->events & EPOLLOUT) &&
2697 !(s->vep.ev.events & EPOLLOUT) && (vcl_session_write_ready (s) > 0))
2699 session_event_t e = { 0 };
2700 e.event_type = SESSION_IO_EVT_TX;
2701 e.session_index = s->session_index;
2702 vec_add1 (wrk->unhandled_evts_vector, e);
2704 s->vep.et_mask = VEP_DEFAULT_ET_MASK;
2706 txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
2707 if (event->events & EPOLLOUT)
2708 svm_fifo_add_want_deq_ntf (txf, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2710 svm_fifo_del_want_deq_ntf (txf, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2711 VDBG (1, "EPOLL_CTL_MOD: vep_sh %u, sh %u, events 0x%x, data 0x%llx!",
2712 vep_handle, session_handle, event->events, event->data.u64);
2716 if (PREDICT_FALSE (!(s->flags & VCL_SESSION_F_IS_VEP_SESSION)))
2718 VDBG (0, "EPOLL_CTL_DEL: %u not a vep session!", session_handle);
2722 else if (PREDICT_FALSE (s->vep.vep_sh != vep_handle))
2724 VDBG (0, "EPOLL_CTL_DEL: sh %u vep_sh (%u) != vep_sh (%u)!",
2725 session_handle, s->vep.vep_sh, vep_handle);
2730 if (s->vep.prev_sh == vep_handle)
2731 vep_session->vep.next_sh = s->vep.next_sh;
2734 vcl_session_t *prev_session;
2735 prev_session = vcl_session_get_w_handle (wrk, s->vep.prev_sh);
2736 if (PREDICT_FALSE (!prev_session))
2738 VDBG (0, "EPOLL_CTL_DEL: Invalid prev_sh (%u) on sh (%u)!",
2739 s->vep.prev_sh, session_handle);
2740 return VPPCOM_EBADFD;
2742 ASSERT (prev_session->vep.next_sh == session_handle);
2743 prev_session->vep.next_sh = s->vep.next_sh;
2745 if (s->vep.next_sh != ~0)
2747 vcl_session_t *next_session;
2748 next_session = vcl_session_get_w_handle (wrk, s->vep.next_sh);
2749 if (PREDICT_FALSE (!next_session))
2751 VDBG (0, "EPOLL_CTL_DEL: Invalid next_sh (%u) on sh (%u)!",
2752 s->vep.next_sh, session_handle);
2753 return VPPCOM_EBADFD;
2755 ASSERT (next_session->vep.prev_sh == session_handle);
2756 next_session->vep.prev_sh = s->vep.prev_sh;
2759 memset (&s->vep, 0, sizeof (s->vep));
2760 s->vep.next_sh = ~0;
2761 s->vep.prev_sh = ~0;
2763 s->flags &= ~VCL_SESSION_F_IS_VEP_SESSION;
2765 txf = vcl_session_is_ct (s) ? s->ct_tx_fifo : s->tx_fifo;
2767 svm_fifo_del_want_deq_ntf (txf, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL);
2769 VDBG (1, "EPOLL_CTL_DEL: vep_idx %u, sh %u!", vep_handle,
2771 vcl_evt (VCL_EVT_EPOLL_CTLDEL, s, vep_sh);
2775 VDBG (0, "Invalid operation (%d)!", op);
2779 vep_verify_epoll_chain (wrk, vep_handle);
2786 vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
2787 struct epoll_event *events, u32 * num_ev)
2789 session_disconnected_msg_t *disconnected_msg;
2790 session_connected_msg_t *connected_msg;
2791 u32 sid = ~0, session_events;
2792 u64 session_evt_data = ~0;
2796 switch (e->event_type)
2798 case SESSION_IO_EVT_RX:
2799 sid = e->session_index;
2800 s = vcl_session_get (wrk, sid);
2801 if (vcl_session_is_closed (s))
2803 vcl_fifo_rx_evt_valid_or_break (s);
2804 session_events = s->vep.ev.events;
2805 if (!(EPOLLIN & s->vep.ev.events)
2806 || (s->flags & VCL_SESSION_F_HAS_RX_EVT))
2809 events[*num_ev].events |= EPOLLIN;
2810 session_evt_data = s->vep.ev.data.u64;
2811 s->flags |= VCL_SESSION_F_HAS_RX_EVT;
2813 case SESSION_IO_EVT_TX:
2814 sid = e->session_index;
2815 s = vcl_session_get (wrk, sid);
2816 if (vcl_session_is_closed (s))
2818 session_events = s->vep.ev.events;
2819 if (!(EPOLLOUT & session_events))
2822 events[*num_ev].events |= EPOLLOUT;
2823 session_evt_data = s->vep.ev.data.u64;
2824 svm_fifo_reset_has_deq_ntf (vcl_session_is_ct (s) ?
2825 s->ct_tx_fifo : s->tx_fifo);
2827 case SESSION_CTRL_EVT_ACCEPTED:
2829 s = vcl_session_accepted (wrk, (session_accepted_msg_t *) e->data);
2831 s = vcl_session_get (wrk, e->session_index);
2834 session_events = s->vep.ev.events;
2835 sid = s->session_index;
2836 if (!(EPOLLIN & session_events))
2839 events[*num_ev].events |= EPOLLIN;
2840 session_evt_data = s->vep.ev.data.u64;
2842 case SESSION_CTRL_EVT_CONNECTED:
2845 connected_msg = (session_connected_msg_t *) e->data;
2846 sid = vcl_session_connected_handler (wrk, connected_msg);
2849 sid = e->session_index;
2850 s = vcl_session_get (wrk, sid);
2851 if (vcl_session_is_closed (s))
2853 session_events = s->vep.ev.events;
2854 /* Generate EPOLLOUT because there's no connected event */
2855 if (!(EPOLLOUT & session_events))
2858 events[*num_ev].events |= EPOLLOUT;
2859 session_evt_data = s->vep.ev.data.u64;
2860 if (s->session_state == VCL_STATE_DETACHED)
2861 events[*num_ev].events |= EPOLLHUP;
2863 case SESSION_CTRL_EVT_DISCONNECTED:
2864 disconnected_msg = (session_disconnected_msg_t *) e->data;
2865 s = vcl_session_disconnected_handler (wrk, disconnected_msg);
2866 if (vcl_session_is_closed (s))
2868 sid = s->session_index;
2869 session_events = s->vep.ev.events;
2871 events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
2872 session_evt_data = s->vep.ev.data.u64;
2874 case SESSION_CTRL_EVT_RESET:
2875 sid = vcl_session_reset_handler (wrk, (session_reset_msg_t *) e->data);
2876 s = vcl_session_get (wrk, sid);
2877 if (vcl_session_is_closed (s))
2879 session_events = s->vep.ev.events;
2881 events[*num_ev].events |= EPOLLHUP | EPOLLRDHUP;
2882 session_evt_data = s->vep.ev.data.u64;
2884 case SESSION_CTRL_EVT_UNLISTEN_REPLY:
2885 vcl_session_unlisten_reply_handler (wrk, e->data);
2887 case SESSION_CTRL_EVT_MIGRATED:
2888 vcl_session_migrated_handler (wrk, e->data);
2890 case SESSION_CTRL_EVT_CLEANUP:
2891 vcl_session_cleanup_handler (wrk, e->data);
2893 case SESSION_CTRL_EVT_REQ_WORKER_UPDATE:
2894 vcl_session_req_worker_update_handler (wrk, e->data);
2896 case SESSION_CTRL_EVT_WORKER_UPDATE_REPLY:
2897 vcl_session_worker_update_reply_handler (wrk, e->data);
2899 case SESSION_CTRL_EVT_APP_ADD_SEGMENT:
2900 vcl_session_app_add_segment_handler (wrk, e->data);
2902 case SESSION_CTRL_EVT_APP_DEL_SEGMENT:
2903 vcl_session_app_del_segment_handler (wrk, e->data);
2905 case SESSION_CTRL_EVT_APP_WRK_RPC:
2906 vcl_worker_rpc_handler (wrk, e->data);
2909 VDBG (0, "unhandled: %u", e->event_type);
2915 events[*num_ev].data.u64 = session_evt_data;
2916 if (EPOLLONESHOT & session_events)
2918 s = vcl_session_get (wrk, sid);
2919 s->vep.ev.events = 0;
2926 vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
2927 struct epoll_event *events, u32 maxevents,
2928 double wait_for_time, u32 * num_ev)
2930 svm_msg_q_msg_t *msg;
2934 if (vec_len (wrk->mq_msg_vector) && svm_msg_q_is_empty (mq))
2935 goto handle_dequeued;
2937 svm_msg_q_lock (mq);
2938 if (svm_msg_q_is_empty (mq))
2942 svm_msg_q_unlock (mq);
2945 else if (wait_for_time < 0)
2947 svm_msg_q_wait (mq);
2951 if (svm_msg_q_timedwait (mq, wait_for_time / 1e3))
2953 svm_msg_q_unlock (mq);
2958 ASSERT (maxevents > *num_ev);
2959 vcl_mq_dequeue_batch (wrk, mq, ~0);
2960 svm_msg_q_unlock (mq);
2963 for (i = 0; i < vec_len (wrk->mq_msg_vector); i++)
2965 msg = vec_elt_at_index (wrk->mq_msg_vector, i);
2966 e = svm_msg_q_msg_data (mq, msg);
2967 if (*num_ev < maxevents)
2968 vcl_epoll_wait_handle_mq_event (wrk, e, events, num_ev);
2970 vcl_handle_mq_event (wrk, e);
2971 svm_msg_q_free_msg (mq, msg);
2973 vec_reset_length (wrk->mq_msg_vector);
2974 vcl_handle_pending_wrk_updates (wrk);
2979 vppcom_epoll_wait_condvar (vcl_worker_t * wrk, struct epoll_event *events,
2980 int maxevents, u32 n_evts, double wait_for_time)
2982 double wait = 0, start = 0, now;
2986 wait = wait_for_time;
2987 start = clib_time_now (&wrk->clib_time);
2992 vcl_epoll_wait_handle_mq (wrk, wrk->app_event_queue, events, maxevents,
2999 now = clib_time_now (&wrk->clib_time);
3000 wait -= (now - start) * 1e3;
3009 vppcom_epoll_wait_eventfd (vcl_worker_t * wrk, struct epoll_event *events,
3010 int maxevents, u32 n_evts, double wait_for_time)
3012 vcl_mq_evt_conn_t *mqc;
3013 int __clib_unused n_read;
3017 vec_validate (wrk->mq_events, pool_elts (wrk->mq_evt_conns));
3019 n_mq_evts = epoll_wait (wrk->mqs_epfd, wrk->mq_events,
3020 vec_len (wrk->mq_events), wait_for_time);
3021 for (i = 0; i < n_mq_evts; i++)
3023 mqc = vcl_mq_evt_conn_get (wrk, wrk->mq_events[i].data.u32);
3024 n_read = read (mqc->mq_fd, &buf, sizeof (buf));
3025 vcl_epoll_wait_handle_mq (wrk, mqc->mq, events, maxevents, 0, &n_evts);
3027 if (!n_evts && n_mq_evts > 0)
3030 return (int) n_evts;
3034 vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
3035 int maxevents, double wait_for_time)
3037 vcl_worker_t *wrk = vcl_worker_get_current ();
3038 vcl_session_t *vep_session;
3042 if (PREDICT_FALSE (maxevents <= 0))
3044 VDBG (0, "ERROR: Invalid maxevents (%d)!", maxevents);
3045 return VPPCOM_EINVAL;
3048 vep_session = vcl_session_get_w_handle (wrk, vep_handle);
3050 return VPPCOM_EBADFD;
3052 if (PREDICT_FALSE (!(vep_session->flags & VCL_SESSION_F_IS_VEP)))
3054 VDBG (0, "ERROR: vep_idx (%u) is not a vep!", vep_handle);
3055 return VPPCOM_EINVAL;
3058 memset (events, 0, sizeof (*events) * maxevents);
3060 if (vec_len (wrk->unhandled_evts_vector))
3062 for (i = 0; i < vec_len (wrk->unhandled_evts_vector); i++)
3064 vcl_epoll_wait_handle_mq_event (wrk, &wrk->unhandled_evts_vector[i],
3066 if (n_evts == maxevents)
3068 vec_delete (wrk->unhandled_evts_vector, i + 1, 0);
3072 vec_reset_length (wrk->unhandled_evts_vector);
3075 if (vcm->cfg.use_mq_eventfd)
3076 return vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
3079 return vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
3084 vppcom_session_attr (uint32_t session_handle, uint32_t op,
3085 void *buffer, uint32_t * buflen)
3087 vcl_worker_t *wrk = vcl_worker_get_current ();
3088 u32 *flags = buffer, tmp_flags = 0;
3089 vppcom_endpt_t *ep = buffer;
3090 vcl_session_t *session;
3093 session = vcl_session_get_w_handle (wrk, session_handle);
3095 return VPPCOM_EBADFD;
3099 case VPPCOM_ATTR_GET_NREAD:
3100 rv = vcl_session_read_ready (session);
3101 VDBG (2, "VPPCOM_ATTR_GET_NREAD: sh %u, nread = %d", session_handle,
3105 case VPPCOM_ATTR_GET_NWRITE:
3106 rv = vcl_session_write_ready (session);
3107 VDBG (2, "VPPCOM_ATTR_GET_NWRITE: sh %u, nwrite = %d", session_handle,
3111 case VPPCOM_ATTR_GET_FLAGS:
3112 if (PREDICT_TRUE (buffer && buflen && (*buflen >= sizeof (*flags))))
3116 (vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK) ?
3118 *buflen = sizeof (*flags);
3119 VDBG (2, "VPPCOM_ATTR_GET_FLAGS: sh %u, flags = 0x%08x, "
3120 "is_nonblocking = %u", session_handle, *flags,
3121 vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3127 case VPPCOM_ATTR_SET_FLAGS:
3128 if (PREDICT_TRUE (buffer && buflen && (*buflen == sizeof (*flags))))
3130 if (*flags & O_NONBLOCK)
3131 vcl_session_set_attr (session, VCL_SESS_ATTR_NONBLOCK);
3133 vcl_session_clear_attr (session, VCL_SESS_ATTR_NONBLOCK);
3135 VDBG (2, "VPPCOM_ATTR_SET_FLAGS: sh %u, flags = 0x%08x,"
3136 " is_nonblocking = %u", session_handle, *flags,
3137 vcl_session_has_attr (session, VCL_SESS_ATTR_NONBLOCK));
3143 case VPPCOM_ATTR_GET_PEER_ADDR:
3144 if (PREDICT_TRUE (buffer && buflen &&
3145 (*buflen >= sizeof (*ep)) && ep->ip))
3147 ep->is_ip4 = session->transport.is_ip4;
3148 ep->port = session->transport.rmt_port;
3149 if (session->transport.is_ip4)
3150 clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3151 sizeof (ip4_address_t));
3153 clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3154 sizeof (ip6_address_t));
3155 *buflen = sizeof (*ep);
3156 VDBG (1, "VPPCOM_ATTR_GET_PEER_ADDR: sh %u, is_ip4 = %u, "
3157 "addr = %U, port %u", session_handle, ep->is_ip4,
3158 format_ip46_address, &session->transport.rmt_ip,
3159 ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3160 clib_net_to_host_u16 (ep->port));
3166 case VPPCOM_ATTR_GET_LCL_ADDR:
3167 if (PREDICT_TRUE (buffer && buflen &&
3168 (*buflen >= sizeof (*ep)) && ep->ip))
3170 ep->is_ip4 = session->transport.is_ip4;
3171 ep->port = session->transport.lcl_port;
3172 if (session->transport.is_ip4)
3173 clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip4,
3174 sizeof (ip4_address_t));
3176 clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip6,
3177 sizeof (ip6_address_t));
3178 *buflen = sizeof (*ep);
3179 VDBG (1, "VPPCOM_ATTR_GET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3180 " port %d", session_handle, ep->is_ip4, format_ip46_address,
3181 &session->transport.lcl_ip,
3182 ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3183 clib_net_to_host_u16 (ep->port));
3189 case VPPCOM_ATTR_SET_LCL_ADDR:
3190 if (PREDICT_TRUE (buffer && buflen &&
3191 (*buflen >= sizeof (*ep)) && ep->ip))
3193 session->transport.is_ip4 = ep->is_ip4;
3194 session->transport.lcl_port = ep->port;
3195 vcl_ip_copy_from_ep (&session->transport.lcl_ip, ep);
3196 *buflen = sizeof (*ep);
3197 VDBG (1, "VPPCOM_ATTR_SET_LCL_ADDR: sh %u, is_ip4 = %u, addr = %U"
3198 " port %d", session_handle, ep->is_ip4, format_ip46_address,
3199 &session->transport.lcl_ip,
3200 ep->is_ip4 ? IP46_TYPE_IP4 : IP46_TYPE_IP6,
3201 clib_net_to_host_u16 (ep->port));
3207 case VPPCOM_ATTR_GET_LIBC_EPFD:
3208 rv = session->libc_epfd;
3209 VDBG (2, "VPPCOM_ATTR_GET_LIBC_EPFD: libc_epfd %d", rv);
3212 case VPPCOM_ATTR_SET_LIBC_EPFD:
3213 if (PREDICT_TRUE (buffer && buflen &&
3214 (*buflen == sizeof (session->libc_epfd))))
3216 session->libc_epfd = *(int *) buffer;
3217 *buflen = sizeof (session->libc_epfd);
3219 VDBG (2, "VPPCOM_ATTR_SET_LIBC_EPFD: libc_epfd %d, buflen %d",
3220 session->libc_epfd, *buflen);
3226 case VPPCOM_ATTR_GET_PROTOCOL:
3227 if (buffer && buflen && (*buflen >= sizeof (int)))
3229 *(int *) buffer = session->session_type;
3230 *buflen = sizeof (int);
3232 VDBG (2, "VPPCOM_ATTR_GET_PROTOCOL: %d (%s), buflen %d",
3233 *(int *) buffer, *(int *) buffer ? "UDP" : "TCP", *buflen);
3239 case VPPCOM_ATTR_GET_LISTEN:
3240 if (buffer && buflen && (*buflen >= sizeof (int)))
3242 *(int *) buffer = vcl_session_has_attr (session,
3243 VCL_SESS_ATTR_LISTEN);
3244 *buflen = sizeof (int);
3246 VDBG (2, "VPPCOM_ATTR_GET_LISTEN: %d, buflen %d", *(int *) buffer,
3253 case VPPCOM_ATTR_GET_ERROR:
3254 if (buffer && buflen && (*buflen >= sizeof (int)))
3256 *(int *) buffer = 0;
3257 *buflen = sizeof (int);
3259 VDBG (2, "VPPCOM_ATTR_GET_ERROR: %d, buflen %d, #VPP-TBD#",
3260 *(int *) buffer, *buflen);
3266 case VPPCOM_ATTR_GET_TX_FIFO_LEN:
3267 if (buffer && buflen && (*buflen >= sizeof (u32)))
3271 *(size_t *) buffer = (session->sndbuf_size ? session->sndbuf_size :
3273 svm_fifo_size (session->tx_fifo) :
3274 vcm->cfg.tx_fifo_size);
3275 *buflen = sizeof (u32);
3277 VDBG (2, "VPPCOM_ATTR_GET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3278 " #VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer,
3285 case VPPCOM_ATTR_SET_TX_FIFO_LEN:
3286 if (buffer && buflen && (*buflen == sizeof (u32)))
3289 session->sndbuf_size = *(u32 *) buffer;
3290 VDBG (2, "VPPCOM_ATTR_SET_TX_FIFO_LEN: %u (0x%x), buflen %d,"
3291 " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3298 case VPPCOM_ATTR_GET_RX_FIFO_LEN:
3299 if (buffer && buflen && (*buflen >= sizeof (u32)))
3303 *(size_t *) buffer = (session->rcvbuf_size ? session->rcvbuf_size :
3305 svm_fifo_size (session->rx_fifo) :
3306 vcm->cfg.rx_fifo_size);
3307 *buflen = sizeof (u32);
3309 VDBG (2, "VPPCOM_ATTR_GET_RX_FIFO_LEN: %u (0x%x), buflen %d, "
3310 "#VPP-TBD#", *(size_t *) buffer, *(size_t *) buffer, *buflen);
3316 case VPPCOM_ATTR_SET_RX_FIFO_LEN:
3317 if (buffer && buflen && (*buflen == sizeof (u32)))
3320 session->rcvbuf_size = *(u32 *) buffer;
3321 VDBG (2, "VPPCOM_ATTR_SET_RX_FIFO_LEN: %u (0x%x), buflen %d,"
3322 " #VPP-TBD#", session->sndbuf_size, session->sndbuf_size,
3329 case VPPCOM_ATTR_GET_REUSEADDR:
3330 if (buffer && buflen && (*buflen >= sizeof (int)))
3333 *(int *) buffer = vcl_session_has_attr (session,
3334 VCL_SESS_ATTR_REUSEADDR);
3335 *buflen = sizeof (int);
3337 VDBG (2, "VPPCOM_ATTR_GET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3338 *(int *) buffer, *buflen);
3344 case VPPCOM_ATTR_SET_REUSEADDR:
3345 if (buffer && buflen && (*buflen == sizeof (int)) &&
3346 !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3349 if (*(int *) buffer)
3350 vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEADDR);
3352 vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEADDR);
3354 VDBG (2, "VPPCOM_ATTR_SET_REUSEADDR: %d, buflen %d, #VPP-TBD#",
3355 vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEADDR),
3362 case VPPCOM_ATTR_GET_REUSEPORT:
3363 if (buffer && buflen && (*buflen >= sizeof (int)))
3366 *(int *) buffer = vcl_session_has_attr (session,
3367 VCL_SESS_ATTR_REUSEPORT);
3368 *buflen = sizeof (int);
3370 VDBG (2, "VPPCOM_ATTR_GET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3371 *(int *) buffer, *buflen);
3377 case VPPCOM_ATTR_SET_REUSEPORT:
3378 if (buffer && buflen && (*buflen == sizeof (int)) &&
3379 !vcl_session_has_attr (session, VCL_SESS_ATTR_LISTEN))
3382 if (*(int *) buffer)
3383 vcl_session_set_attr (session, VCL_SESS_ATTR_REUSEPORT);
3385 vcl_session_clear_attr (session, VCL_SESS_ATTR_REUSEPORT);
3387 VDBG (2, "VPPCOM_ATTR_SET_REUSEPORT: %d, buflen %d, #VPP-TBD#",
3388 vcl_session_has_attr (session, VCL_SESS_ATTR_REUSEPORT),
3395 case VPPCOM_ATTR_GET_BROADCAST:
3396 if (buffer && buflen && (*buflen >= sizeof (int)))
3399 *(int *) buffer = vcl_session_has_attr (session,
3400 VCL_SESS_ATTR_BROADCAST);
3401 *buflen = sizeof (int);
3403 VDBG (2, "VPPCOM_ATTR_GET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3404 *(int *) buffer, *buflen);
3410 case VPPCOM_ATTR_SET_BROADCAST:
3411 if (buffer && buflen && (*buflen == sizeof (int)))
3414 if (*(int *) buffer)
3415 vcl_session_set_attr (session, VCL_SESS_ATTR_BROADCAST);
3417 vcl_session_clear_attr (session, VCL_SESS_ATTR_BROADCAST);
3419 VDBG (2, "VPPCOM_ATTR_SET_BROADCAST: %d, buflen %d, #VPP-TBD#",
3420 vcl_session_has_attr (session, VCL_SESS_ATTR_BROADCAST),
3427 case VPPCOM_ATTR_GET_V6ONLY:
3428 if (buffer && buflen && (*buflen >= sizeof (int)))
3431 *(int *) buffer = vcl_session_has_attr (session,
3432 VCL_SESS_ATTR_V6ONLY);
3433 *buflen = sizeof (int);
3435 VDBG (2, "VPPCOM_ATTR_GET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3436 *(int *) buffer, *buflen);
3442 case VPPCOM_ATTR_SET_V6ONLY:
3443 if (buffer && buflen && (*buflen == sizeof (int)))
3446 if (*(int *) buffer)
3447 vcl_session_set_attr (session, VCL_SESS_ATTR_V6ONLY);
3449 vcl_session_clear_attr (session, VCL_SESS_ATTR_V6ONLY);
3451 VDBG (2, "VPPCOM_ATTR_SET_V6ONLY: %d, buflen %d, #VPP-TBD#",
3452 vcl_session_has_attr (session, VCL_SESS_ATTR_V6ONLY),
3459 case VPPCOM_ATTR_GET_KEEPALIVE:
3460 if (buffer && buflen && (*buflen >= sizeof (int)))
3463 *(int *) buffer = vcl_session_has_attr (session,
3464 VCL_SESS_ATTR_KEEPALIVE);
3465 *buflen = sizeof (int);
3467 VDBG (2, "VPPCOM_ATTR_GET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3468 *(int *) buffer, *buflen);
3474 case VPPCOM_ATTR_SET_KEEPALIVE:
3475 if (buffer && buflen && (*buflen == sizeof (int)))
3478 if (*(int *) buffer)
3479 vcl_session_set_attr (session, VCL_SESS_ATTR_KEEPALIVE);
3481 vcl_session_clear_attr (session, VCL_SESS_ATTR_KEEPALIVE);
3483 VDBG (2, "VPPCOM_ATTR_SET_KEEPALIVE: %d, buflen %d, #VPP-TBD#",
3484 vcl_session_has_attr (session, VCL_SESS_ATTR_KEEPALIVE),
3491 case VPPCOM_ATTR_GET_TCP_NODELAY:
3492 if (buffer && buflen && (*buflen >= sizeof (int)))
3495 *(int *) buffer = vcl_session_has_attr (session,
3496 VCL_SESS_ATTR_TCP_NODELAY);
3497 *buflen = sizeof (int);
3499 VDBG (2, "VPPCOM_ATTR_GET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3500 *(int *) buffer, *buflen);
3506 case VPPCOM_ATTR_SET_TCP_NODELAY:
3507 if (buffer && buflen && (*buflen == sizeof (int)))
3510 if (*(int *) buffer)
3511 vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
3513 vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_NODELAY);
3515 VDBG (2, "VPPCOM_ATTR_SET_TCP_NODELAY: %d, buflen %d, #VPP-TBD#",
3516 vcl_session_has_attr (session, VCL_SESS_ATTR_TCP_NODELAY),
3523 case VPPCOM_ATTR_GET_TCP_KEEPIDLE:
3524 if (buffer && buflen && (*buflen >= sizeof (int)))
3527 *(int *) buffer = vcl_session_has_attr (session,
3528 VCL_SESS_ATTR_TCP_KEEPIDLE);
3529 *buflen = sizeof (int);
3531 VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3532 *(int *) buffer, *buflen);
3538 case VPPCOM_ATTR_SET_TCP_KEEPIDLE:
3539 if (buffer && buflen && (*buflen == sizeof (int)))
3542 if (*(int *) buffer)
3543 vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
3545 vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPIDLE);
3547 VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPIDLE: %d, buflen %d, #VPP-TBD#",
3548 vcl_session_has_attr (session,
3549 VCL_SESS_ATTR_TCP_KEEPIDLE), *buflen);
3555 case VPPCOM_ATTR_GET_TCP_KEEPINTVL:
3556 if (buffer && buflen && (*buflen >= sizeof (int)))
3559 *(int *) buffer = vcl_session_has_attr (session,
3560 VCL_SESS_ATTR_TCP_KEEPINTVL);
3561 *buflen = sizeof (int);
3563 VDBG (2, "VPPCOM_ATTR_GET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
3564 *(int *) buffer, *buflen);
3570 case VPPCOM_ATTR_SET_TCP_KEEPINTVL:
3571 if (buffer && buflen && (*buflen == sizeof (int)))
3574 if (*(int *) buffer)
3575 vcl_session_set_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
3577 vcl_session_clear_attr (session, VCL_SESS_ATTR_TCP_KEEPINTVL);
3579 VDBG (2, "VPPCOM_ATTR_SET_TCP_KEEPINTVL: %d, buflen %d, #VPP-TBD#",
3580 vcl_session_has_attr (session,
3581 VCL_SESS_ATTR_TCP_KEEPINTVL), *buflen);
3587 case VPPCOM_ATTR_GET_TCP_USER_MSS:
3588 if (buffer && buflen && (*buflen >= sizeof (u32)))
3591 *(u32 *) buffer = session->user_mss;
3592 *buflen = sizeof (int);
3594 VDBG (2, "VPPCOM_ATTR_GET_TCP_USER_MSS: %d, buflen %d, #VPP-TBD#",
3595 *(int *) buffer, *buflen);
3601 case VPPCOM_ATTR_SET_TCP_USER_MSS:
3602 if (buffer && buflen && (*buflen == sizeof (u32)))
3605 session->user_mss = *(u32 *) buffer;
3607 VDBG (2, "VPPCOM_ATTR_SET_TCP_USER_MSS: %u, buflen %d, #VPP-TBD#",
3608 session->user_mss, *buflen);
3614 case VPPCOM_ATTR_SET_SHUT:
3615 if (*flags == SHUT_RD || *flags == SHUT_RDWR)
3616 vcl_session_set_attr (session, VCL_SESS_ATTR_SHUT_RD);
3617 if (*flags == SHUT_WR || *flags == SHUT_RDWR)
3618 vcl_session_set_attr (session, VCL_SESS_ATTR_SHUT_WR);
3621 case VPPCOM_ATTR_GET_SHUT:
3622 if (vcl_session_has_attr (session, VCL_SESS_ATTR_SHUT_RD))
3624 if (vcl_session_has_attr (session, VCL_SESS_ATTR_SHUT_WR))
3627 *(int *) buffer = SHUT_RD;
3628 else if (tmp_flags == 2)
3629 *(int *) buffer = SHUT_WR;
3630 else if (tmp_flags == 3)
3631 *(int *) buffer = SHUT_RDWR;
3632 *buflen = sizeof (int);
3635 case VPPCOM_ATTR_SET_CONNECTED:
3636 session->flags |= VCL_SESSION_F_CONNECTED;
3639 case VPPCOM_ATTR_SET_CKPAIR:
3640 if (!(buffer && buflen && (*buflen == sizeof (int))) ||
3641 !vcl_session_has_crypto (session))
3646 session->ckpair_index = *(uint32_t *) buffer;
3658 vppcom_session_recvfrom (uint32_t session_handle, void *buffer,
3659 uint32_t buflen, int flags, vppcom_endpt_t * ep)
3661 vcl_worker_t *wrk = vcl_worker_get_current ();
3662 vcl_session_t *session;
3666 rv = vppcom_session_read (session_handle, buffer, buflen);
3667 else if (flags & MSG_PEEK)
3668 rv = vppcom_session_peek (session_handle, buffer, buflen);
3671 VDBG (0, "Unsupport flags for recvfrom %d", flags);
3672 return VPPCOM_EAFNOSUPPORT;
3677 session = vcl_session_get_w_handle (wrk, session_handle);
3678 if (session->transport.is_ip4)
3679 clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4,
3680 sizeof (ip4_address_t));
3682 clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6,
3683 sizeof (ip6_address_t));
3684 ep->is_ip4 = session->transport.is_ip4;
3685 ep->port = session->transport.rmt_port;
3692 vppcom_session_sendto (uint32_t session_handle, void *buffer,
3693 uint32_t buflen, int flags, vppcom_endpt_t * ep)
3695 vcl_worker_t *wrk = vcl_worker_get_current ();
3698 s = vcl_session_get_w_handle (wrk, session_handle);
3700 return VPPCOM_EBADFD;
3703 return VPPCOM_EINVAL;
3707 if (!vcl_session_is_cl (s))
3708 return VPPCOM_EINVAL;
3710 /* Session not connected/bound in vpp. Create it by 'connecting' it */
3711 if (PREDICT_FALSE (s->session_state == VCL_STATE_CLOSED))
3713 u32 session_index = s->session_index;
3714 f64 timeout = vcm->cfg.session_timeout;
3717 vcl_send_session_connect (wrk, s);
3718 rv = vppcom_wait_for_session_state_change (session_index,
3723 s = vcl_session_get (wrk, session_index);
3726 s->transport.is_ip4 = ep->is_ip4;
3727 s->transport.rmt_port = ep->port;
3728 vcl_ip_copy_from_ep (&s->transport.rmt_ip, ep);
3733 // TBD check the flags and do the right thing
3734 VDBG (2, "handling flags 0x%u (%d) not implemented yet.", flags, flags);
3737 return (vppcom_session_write_inline (wrk, s, buffer, buflen, 1,
3738 s->is_dgram ? 1 : 0));
3742 vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
3744 vcl_worker_t *wrk = vcl_worker_get_current ();
3745 f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
3746 u32 i, keep_trying = 1;
3747 svm_msg_q_msg_t msg;
3751 VDBG (3, "vp %p, nsids %u, wait_for_time %f", vp, n_sids, wait_for_time);
3754 return VPPCOM_EFAULT;
3758 vcl_session_t *session;
3760 /* Dequeue all events and drop all unhandled io events */
3761 while (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0) == 0)
3763 e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
3764 vcl_handle_mq_event (wrk, e);
3765 svm_msg_q_free_msg (wrk->app_event_queue, &msg);
3767 vec_reset_length (wrk->unhandled_evts_vector);
3769 for (i = 0; i < n_sids; i++)
3771 session = vcl_session_get (wrk, vp[i].sh);
3774 vp[i].revents = POLLHUP;
3781 if (POLLIN & vp[i].events)
3783 rv = vcl_session_read_ready (session);
3786 vp[i].revents |= POLLIN;
3793 case VPPCOM_ECONNRESET:
3794 vp[i].revents = POLLHUP;
3798 vp[i].revents = POLLERR;
3805 if (POLLOUT & vp[i].events)
3807 rv = vcl_session_write_ready (session);
3810 vp[i].revents |= POLLOUT;
3817 case VPPCOM_ECONNRESET:
3818 vp[i].revents = POLLHUP;
3822 vp[i].revents = POLLERR;
3829 if (0) // Note "done:" label used by VCL_SESSION_LOCK_AND_GET()
3831 vp[i].revents = POLLNVAL;
3835 if (wait_for_time != -1)
3836 keep_trying = (clib_time_now (&wrk->clib_time) <= timeout) ? 1 : 0;
3838 while ((num_ev == 0) && keep_trying);
3844 vppcom_mq_epoll_fd (void)
3846 vcl_worker_t *wrk = vcl_worker_get_current ();
3847 return wrk->mqs_epfd;
3851 vppcom_session_index (vcl_session_handle_t session_handle)
3853 return session_handle & 0xFFFFFF;
3857 vppcom_session_worker (vcl_session_handle_t session_handle)
3859 return session_handle >> 24;
3863 vppcom_worker_register (void)
3865 if (!vcl_worker_alloc_and_init ())
3866 return VPPCOM_EEXIST;
3868 if (vcl_worker_register_with_vpp ())
3869 return VPPCOM_EEXIST;
3875 vppcom_worker_unregister (void)
3877 vcl_worker_cleanup (vcl_worker_get_current (), 1 /* notify vpp */ );
3878 vcl_set_worker_index (~0);
3882 vppcom_worker_index_set (int index)
3884 vcl_set_worker_index (index);
3888 vppcom_worker_index (void)
3890 return vcl_get_worker_index ();
3894 vppcom_worker_mqs_epfd (void)
3896 vcl_worker_t *wrk = vcl_worker_get_current ();
3897 if (!vcm->cfg.use_mq_eventfd)
3899 return wrk->mqs_epfd;
3903 vppcom_session_is_connectable_listener (uint32_t session_handle)
3905 vcl_session_t *session;
3906 vcl_worker_t *wrk = vcl_worker_get_current ();
3907 session = vcl_session_get_w_handle (wrk, session_handle);
3909 return VPPCOM_EBADFD;
3910 return vcl_session_is_connectable_listener (wrk, session);
3914 vppcom_session_listener (uint32_t session_handle)
3916 vcl_worker_t *wrk = vcl_worker_get_current ();
3917 vcl_session_t *listen_session, *session;
3918 session = vcl_session_get_w_handle (wrk, session_handle);
3920 return VPPCOM_EBADFD;
3921 if (session->listener_index == VCL_INVALID_SESSION_INDEX)
3922 return VPPCOM_EBADFD;
3923 listen_session = vcl_session_get_w_handle (wrk, session->listener_index);
3924 if (!listen_session)
3925 return VPPCOM_EBADFD;
3926 return vcl_session_handle (listen_session);
3930 vppcom_session_n_accepted (uint32_t session_handle)
3932 vcl_worker_t *wrk = vcl_worker_get_current ();
3933 vcl_session_t *session = vcl_session_get_w_handle (wrk, session_handle);
3935 return VPPCOM_EBADFD;
3936 return session->n_accepted_sessions;
3940 vppcom_proto_str (vppcom_proto_t proto)
3942 char const *proto_str;
3946 case VPPCOM_PROTO_TCP:
3949 case VPPCOM_PROTO_UDP:
3952 case VPPCOM_PROTO_TLS:
3955 case VPPCOM_PROTO_QUIC:
3959 proto_str = "UNKNOWN";
3966 vppcom_retval_str (int retval)
3977 st = "VPPCOM_EAGAIN";
3981 st = "VPPCOM_EFAULT";
3985 st = "VPPCOM_ENOMEM";
3989 st = "VPPCOM_EINVAL";
3993 st = "VPPCOM_EBADFD";
3996 case VPPCOM_EAFNOSUPPORT:
3997 st = "VPPCOM_EAFNOSUPPORT";
4000 case VPPCOM_ECONNABORTED:
4001 st = "VPPCOM_ECONNABORTED";
4004 case VPPCOM_ECONNRESET:
4005 st = "VPPCOM_ECONNRESET";
4008 case VPPCOM_ENOTCONN:
4009 st = "VPPCOM_ENOTCONN";
4012 case VPPCOM_ECONNREFUSED:
4013 st = "VPPCOM_ECONNREFUSED";
4016 case VPPCOM_ETIMEDOUT:
4017 st = "VPPCOM_ETIMEDOUT";
4021 st = "UNKNOWN_STATE";
4029 vppcom_add_cert_key_pair (vppcom_cert_key_pair_t *ckpair)
4031 if (vcm->cfg.vpp_app_socket_api)
4033 clib_warning ("not supported");
4034 return VPPCOM_EINVAL;
4036 return vcl_bapi_add_cert_key_pair (ckpair);
4040 vppcom_del_cert_key_pair (uint32_t ckpair_index)
4042 if (vcm->cfg.vpp_app_socket_api)
4044 clib_warning ("not supported");
4045 return VPPCOM_EINVAL;
4047 return vcl_bapi_del_cert_key_pair (ckpair_index);
4051 * fd.io coding-style-patch-verification: ON
4054 * eval: (c-set-style "gnu")