2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
19 #include <vlib/vlib.h>
20 #include <vlibapi/api.h>
21 #include <vlibmemory/api.h>
22 #include <vlibmemory/memory_api.h>
24 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
26 #define vl_typedefs /* define message structures */
27 #include <vlibmemory/vl_memory_api_h.h>
30 /* instantiate all the print functions we know about */
32 #include <vlibmemory/vl_memory_api_h.h>
35 /* instantiate all the endian swap functions we know about */
37 #include <vlibmemory/vl_memory_api_h.h>
40 volatile int **vl_api_queue_cursizes;
43 memclnt_queue_callback (vlib_main_t * vm)
46 api_main_t *am = vlibapi_get_main ();
47 int have_pending_rpcs;
49 if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) !=
50 1 + vec_len (am->vlib_private_rps)))
52 vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
58 q = shmem_hdr->vl_input_queue;
62 vec_add1 (vl_api_queue_cursizes, &q->cursize);
64 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
66 svm_region_t *vlib_rp = am->vlib_private_rps[i];
68 shmem_hdr = (void *) vlib_rp->user_ctx;
69 q = shmem_hdr->vl_input_queue;
70 vec_add1 (vl_api_queue_cursizes, &q->cursize);
74 for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
76 if (*vl_api_queue_cursizes[i])
78 vm->queue_signal_pending = 1;
79 vm->api_queue_nonempty = 1;
80 vlib_process_signal_event (vm, vl_api_clnt_node.index,
81 /* event_type */ QUEUE_SIGNAL_EVENT,
87 clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
88 have_pending_rpcs = vec_len (vm->pending_rpc_requests) > 0;
89 clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
91 if (have_pending_rpcs)
93 vm->queue_signal_pending = 1;
94 vm->api_queue_nonempty = 1;
95 vlib_process_signal_event (vm, vl_api_clnt_node.index,
96 /* event_type */ QUEUE_SIGNAL_EVENT,
102 * vl_api_memclnt_create_internal
105 vl_api_memclnt_create_internal (char *name, svm_queue_t * q)
107 vl_api_registration_t **regpp;
108 vl_api_registration_t *regp;
110 api_main_t *am = vlibapi_get_main ();
112 ASSERT (vlib_get_thread_index () == 0);
113 pool_get (am->vl_clients, regpp);
116 oldheap = vl_msg_push_heap ();
117 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
120 clib_memset (regp, 0, sizeof (*regp));
121 regp->registration_type = REGISTRATION_TYPE_SHMEM;
122 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
123 regp->vlib_rp = am->vlib_rp;
124 regp->shmem_hdr = am->shmem_hdr;
126 regp->vl_input_queue = q;
127 regp->name = format (0, "%s%c", name, 0);
129 vl_msg_pop_heap (oldheap);
130 return vl_msg_api_handle_from_index_and_epoch
131 (regp->vl_api_registration_pool_index,
132 am->shmem_hdr->application_restarts);
136 * vl_api_memclnt_create_t_handler
139 vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
141 vl_api_registration_t **regpp;
142 vl_api_registration_t *regp;
143 vl_api_memclnt_create_reply_t *rp;
147 api_main_t *am = vlibapi_get_main ();
151 * This is tortured. Maintain a vlib-address-space private
152 * pool of client registrations. We use the shared-memory virtual
153 * address of client structure as a handle, to allow direct
154 * manipulation of context quota vbls from the client library.
156 * This scheme causes trouble w/ API message trace replay, since
157 * some random VA from clib_mem_alloc() certainly won't
158 * occur in the Linux sim. The (very) few places
159 * that care need to use the pool index.
161 * Putting the registration object(s) into a pool in shared memory and
162 * using the pool index as a handle seems like a great idea.
163 * Unfortunately, each and every reference to that pool would need
164 * to be protected by a mutex:
168 * convert pool index to
176 pool_get (am->vl_clients, regpp);
178 oldheap = vl_msg_push_heap ();
179 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
182 clib_memset (regp, 0, sizeof (*regp));
183 regp->registration_type = REGISTRATION_TYPE_SHMEM;
184 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
185 regp->vlib_rp = am->vlib_rp;
186 regp->shmem_hdr = am->shmem_hdr;
187 regp->clib_file_index = am->shmem_hdr->clib_file_index;
189 q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
190 VL_MSG_API_SVM_QUEUE_UNPOISON (q);
192 regp->name = format (0, "%s", mp->name);
193 vec_add1 (regp->name, 0);
194 regp->keepalive = true;
196 if (am->serialized_message_table_in_shmem == 0)
197 am->serialized_message_table_in_shmem =
198 vl_api_serialize_message_table (am, 0);
200 if (am->vlib_rp != am->vlib_primary_rp)
201 msg_table = vl_api_serialize_message_table (am, 0);
203 msg_table = am->serialized_message_table_in_shmem;
205 vl_msg_pop_heap (oldheap);
207 rp = vl_msg_api_alloc (sizeof (*rp));
208 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
209 rp->handle = (uword) regp;
210 rp->index = vl_msg_api_handle_from_index_and_epoch
211 (regp->vl_api_registration_pool_index,
212 am->shmem_hdr->application_restarts);
213 rp->context = mp->context;
214 rp->response = ntohl (rv);
215 rp->message_table = pointer_to_uword (msg_table);
217 vl_msg_api_send_shmem (q, (u8 *) & rp);
221 vl_api_memclnt_create_v2_t_handler (vl_api_memclnt_create_v2_t *mp)
223 vl_api_registration_t **regpp;
224 vl_api_registration_t *regp;
225 vl_api_memclnt_create_v2_reply_t *rp;
229 api_main_t *am = vlibapi_get_main ();
233 * This is tortured. Maintain a vlib-address-space private
234 * pool of client registrations. We use the shared-memory virtual
235 * address of client structure as a handle, to allow direct
236 * manipulation of context quota vbls from the client library.
238 * This scheme causes trouble w/ API message trace replay, since
239 * some random VA from clib_mem_alloc() certainly won't
240 * occur in the Linux sim. The (very) few places
241 * that care need to use the pool index.
243 * Putting the registration object(s) into a pool in shared memory and
244 * using the pool index as a handle seems like a great idea.
245 * Unfortunately, each and every reference to that pool would need
246 * to be protected by a mutex:
250 * convert pool index to
258 pool_get (am->vl_clients, regpp);
260 oldheap = vl_msg_push_heap ();
261 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
264 clib_memset (regp, 0, sizeof (*regp));
265 regp->registration_type = REGISTRATION_TYPE_SHMEM;
266 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
267 regp->vlib_rp = am->vlib_rp;
268 regp->shmem_hdr = am->shmem_hdr;
269 regp->clib_file_index = am->shmem_hdr->clib_file_index;
271 q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
272 VL_MSG_API_SVM_QUEUE_UNPOISON (q);
274 regp->name = format (0, "%s", mp->name);
275 vec_add1 (regp->name, 0);
276 regp->keepalive = mp->keepalive;
278 if (am->serialized_message_table_in_shmem == 0)
279 am->serialized_message_table_in_shmem =
280 vl_api_serialize_message_table (am, 0);
282 if (am->vlib_rp != am->vlib_primary_rp)
283 msg_table = vl_api_serialize_message_table (am, 0);
285 msg_table = am->serialized_message_table_in_shmem;
287 vl_msg_pop_heap (oldheap);
289 rp = vl_msg_api_alloc (sizeof (*rp));
290 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_V2_REPLY);
291 rp->handle = (uword) regp;
292 rp->index = vl_msg_api_handle_from_index_and_epoch (
293 regp->vl_api_registration_pool_index, am->shmem_hdr->application_restarts);
294 rp->context = mp->context;
295 rp->response = ntohl (rv);
296 rp->message_table = pointer_to_uword (msg_table);
298 vl_msg_api_send_shmem (q, (u8 *) &rp);
302 vl_api_call_reaper_functions (u32 client_index)
304 clib_error_t *error = 0;
305 _vl_msg_api_function_list_elt_t *i;
307 i = vlibapi_get_main ()->reaper_function_registrations;
310 error = i->f (client_index);
312 clib_error_report (error);
313 i = i->next_init_function;
318 * vl_api_memclnt_delete_t_handler
321 vl_api_memclnt_delete_t_handler (vl_api_memclnt_delete_t * mp)
323 vl_api_registration_t **regpp;
324 vl_api_registration_t *regp;
325 vl_api_memclnt_delete_reply_t *rp;
327 api_main_t *am = vlibapi_get_main ();
328 u32 handle, client_index, epoch;
332 vl_api_call_reaper_functions (handle);
334 epoch = vl_msg_api_handle_get_epoch (handle);
335 client_index = vl_msg_api_handle_get_index (handle);
337 if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
340 ("Stale clnt delete index %d old epoch %d cur epoch %d",
342 (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK));
346 regpp = pool_elt_at_index (am->vl_clients, client_index);
348 if (!pool_is_free (am->vl_clients, regpp))
352 int private_registration = 0;
354 /* Send reply unless client asked us to do the cleanup */
358 * Note: the API message handling path will set am->vlib_rp
359 * as appropriate for pairwise / private memory segments
361 rp = vl_msg_api_alloc (sizeof (*rp));
362 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
363 rp->handle = mp->handle;
366 vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
367 if (client_index != regp->vl_api_registration_pool_index)
369 clib_warning ("mismatch client_index %d pool_index %d",
371 regp->vl_api_registration_pool_index);
372 vl_msg_api_free (rp);
377 /* No dangling references, please */
380 /* For horizontal scaling, add a hash table... */
381 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
383 /* Is this a pairwise / private API segment? */
384 if (am->vlib_private_rps[i] == am->vlib_rp)
386 /* Note: account for the memfd header page */
387 uword virtual_base = am->vlib_rp->virtual_base - MMAP_PAGESIZE;
388 uword virtual_size = am->vlib_rp->virtual_size + MMAP_PAGESIZE;
391 * Kill the registration pool element before we make
392 * the index vanish forever
394 pool_put_index (am->vl_clients,
395 regp->vl_api_registration_pool_index);
397 vec_delete (am->vlib_private_rps, 1, i);
398 /* Kill it, accounting for the memfd header page */
399 if (munmap ((void *) virtual_base, virtual_size) < 0)
400 clib_unix_warning ("munmap");
401 /* Reset the queue-length-address cache */
402 vec_reset_length (vl_api_queue_cursizes);
403 private_registration = 1;
408 if (private_registration == 0)
410 pool_put_index (am->vl_clients,
411 regp->vl_api_registration_pool_index);
412 oldheap = vl_msg_push_heap ();
414 svm_queue_free (regp->vl_input_queue);
415 vec_free (regp->name);
416 /* Poison the old registration */
417 clib_memset (regp, 0xF1, sizeof (*regp));
418 clib_mem_free (regp);
419 vl_msg_pop_heap (oldheap);
421 * These messages must be freed manually, since they're set up
422 * as "bounce" messages. In the private_registration == 1 case,
423 * we kill the shared-memory segment which contains the message
426 vl_msg_api_free (mp);
431 clib_warning ("unknown client ID %d", mp->index);
436 * client answered a ping, stave off the grim reaper...
439 vl_api_memclnt_keepalive_reply_t_handler
440 (vl_api_memclnt_keepalive_reply_t * mp)
442 vl_api_registration_t *regp;
443 vlib_main_t *vm = vlib_get_main ();
445 regp = vl_api_client_index_to_registration (mp->context);
448 regp->last_heard = vlib_time_now (vm);
449 regp->unanswered_pings = 0;
452 clib_warning ("BUG: anonymous memclnt_keepalive_reply");
456 * We can send ourselves these messages if someone uses the
457 * builtin binary api test tool...
460 vl_api_memclnt_keepalive_t_handler (vl_api_memclnt_keepalive_t * mp)
462 vl_api_memclnt_keepalive_reply_t *rmp;
464 vl_shmem_hdr_t *shmem_hdr;
466 am = vlibapi_get_main ();
467 shmem_hdr = am->shmem_hdr;
469 rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
470 clib_memset (rmp, 0, sizeof (*rmp));
471 rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
472 rmp->context = mp->context;
473 vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
477 * To avoid filling the API trace buffer with boring messages,
478 * don't trace memclnt_keepalive[_reply] msgs
481 #define foreach_vlib_api_msg \
482 _ (MEMCLNT_CREATE, memclnt_create, 0) \
483 _ (MEMCLNT_CREATE_V2, memclnt_create_v2, 0) \
484 _ (MEMCLNT_DELETE, memclnt_delete, 0) \
485 _ (MEMCLNT_KEEPALIVE, memclnt_keepalive, 0) \
486 _ (MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
492 vl_mem_api_init (const char *region_name)
495 api_main_t *am = vlibapi_get_main ();
496 vl_msg_api_msg_config_t cfg;
497 vl_msg_api_msg_config_t *c = &cfg;
499 vlib_main_t *vm = vlib_get_main ();
501 clib_memset (c, 0, sizeof (*c));
503 if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
509 c->id = VL_API_##N; \
511 c->handler = vl_api_##n##_t_handler; \
512 c->endian = vl_api_##n##_t_endian; \
513 c->format_fn = vl_api_##n##_t_format; \
514 c->size = sizeof (vl_api_##n##_t); \
515 c->traced = t; /* trace, so these msgs print */ \
516 c->replay = 0; /* don't replay client create/delete msgs */ \
517 c->message_bounce = 0; /* don't bounce this message */ \
518 vl_msg_api_config (c); \
522 foreach_vlib_api_msg;
525 #define vl_msg_name_crc_list
526 #include <vlibmemory/memclnt.api.h>
527 #undef vl_msg_name_crc_list
529 #define _(id, n, crc) vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id);
530 foreach_vl_msg_name_crc_memclnt;
534 * special-case freeing of memclnt_delete messages, so we can
535 * simply munmap pairwise / private API segments...
537 am->msg_data[VL_API_MEMCLNT_DELETE].bounce = 1;
538 vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE_REPLY, 1);
539 vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE, 1);
541 vlib_set_queue_signal_callback (vm, memclnt_queue_callback);
544 ASSERT (shm && shm->vl_input_queue);
546 /* Make a note so we can always find the primary region easily */
547 am->vlib_primary_rp = am->vlib_rp;
553 map_api_segment_init (vlib_main_t * vm)
555 api_main_t *am = vlibapi_get_main ();
558 if ((rv = vl_mem_api_init (am->region_name)) < 0)
560 return clib_error_return (0, "vl_mem_api_init (%s) failed",
567 send_memclnt_keepalive (vl_api_registration_t * regp, f64 now)
569 vl_api_memclnt_keepalive_t *mp;
571 api_main_t *am = vlibapi_get_main ();
573 q = regp->vl_input_queue;
576 * If the queue head is moving, assume that the client is processing
577 * messages and skip the ping. This heuristic may fail if the queue
578 * is in the same position as last time, net of wrapping; in which
579 * case, the client will receive a keepalive.
581 if (regp->last_queue_head != q->head)
583 regp->last_heard = now;
584 regp->unanswered_pings = 0;
585 regp->last_queue_head = q->head;
590 * push/pop shared memory segment, so this routine
591 * will work with "normal" as well as "private segment"
595 mp = vl_mem_api_alloc_as_if_client_w_reg (regp, sizeof (*mp));
596 clib_memset (mp, 0, sizeof (*mp));
597 mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
598 mp->context = mp->client_index =
599 vl_msg_api_handle_from_index_and_epoch
600 (regp->vl_api_registration_pool_index,
601 am->shmem_hdr->application_restarts);
603 regp->unanswered_pings++;
605 /* Failure-to-send due to a stuffed queue is absolutely expected */
606 if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
607 vl_msg_api_free_w_region (regp->vlib_rp, mp);
611 vl_mem_send_client_keepalive_w_reg (api_main_t * am, f64 now,
612 vl_api_registration_t ** regpp,
614 u32 ** confused_indices)
616 vl_api_registration_t *regp = *regpp;
619 /* If we haven't heard from this client recently... */
620 if (regp->last_heard < (now - 10.0))
622 if (regp->unanswered_pings == 2)
625 q = regp->vl_input_queue;
626 if (kill (q->consumer_pid, 0) >= 0)
628 clib_warning ("REAPER: lazy binary API client '%s'",
630 regp->unanswered_pings = 0;
631 regp->last_heard = now;
635 clib_warning ("REAPER: binary API client '%s' died",
637 vec_add1 (*dead_indices, regpp - am->vl_clients);
641 send_memclnt_keepalive (regp, now);
644 regp->unanswered_pings = 0;
648 clib_warning ("NULL client registration index %d",
649 regpp - am->vl_clients);
650 vec_add1 (*confused_indices, regpp - am->vl_clients);
655 vl_mem_api_dead_client_scan (api_main_t * am, vl_shmem_hdr_t * shm, f64 now)
657 vl_api_registration_t **regpp;
658 static u32 *dead_indices;
659 static u32 *confused_indices;
661 vec_reset_length (dead_indices);
662 vec_reset_length (confused_indices);
664 pool_foreach (regpp, am->vl_clients) {
665 if (!(*regpp)->keepalive)
667 vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
671 /* This should "never happen," but if it does, fix it... */
672 if (PREDICT_FALSE (vec_len (confused_indices) > 0))
675 for (i = 0; i < vec_len (confused_indices); i++)
677 pool_put_index (am->vl_clients, confused_indices[i]);
681 if (PREDICT_FALSE (vec_len (dead_indices) > 0))
686 /* Allow the application to clean up its registrations */
687 for (i = 0; i < vec_len (dead_indices); i++)
689 regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
694 handle = vl_msg_api_handle_from_index_and_epoch
695 (dead_indices[i], shm->application_restarts);
696 vl_api_call_reaper_functions (handle);
700 oldheap = vl_msg_push_heap ();
702 for (i = 0; i < vec_len (dead_indices); i++)
704 regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
707 /* Is this a pairwise SVM segment? */
708 if ((*regpp)->vlib_rp != am->vlib_rp)
711 svm_region_t *dead_rp = (*regpp)->vlib_rp;
712 /* Note: account for the memfd header page */
713 uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
714 uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
716 /* For horizontal scaling, add a hash table... */
717 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
718 if (am->vlib_private_rps[i] == dead_rp)
720 vec_delete (am->vlib_private_rps, 1, i);
723 svm_pop_heap (oldheap);
724 clib_warning ("private rp %llx AWOL", dead_rp);
725 oldheap = svm_push_data_heap (am->vlib_rp);
728 /* Kill it, accounting for the memfd header page */
729 svm_pop_heap (oldheap);
730 if (munmap ((void *) virtual_base, virtual_size) < 0)
731 clib_unix_warning ("munmap");
732 /* Reset the queue-length-address cache */
733 vec_reset_length (vl_api_queue_cursizes);
734 oldheap = svm_push_data_heap (am->vlib_rp);
738 /* Poison the old registration */
739 clib_memset (*regpp, 0xF3, sizeof (**regpp));
740 clib_mem_free (*regpp);
742 /* no dangling references, please */
747 svm_pop_heap (oldheap);
748 clib_warning ("Duplicate free, client index %d",
749 regpp - am->vl_clients);
750 oldheap = svm_push_data_heap (am->vlib_rp);
754 svm_client_scan_this_region_nolock (am->vlib_rp);
756 vl_msg_pop_heap (oldheap);
757 for (i = 0; i < vec_len (dead_indices); i++)
758 pool_put_index (am->vl_clients, dead_indices[i]);
762 void (*vl_mem_api_fuzz_hook) (u16, void *);
764 /* This is only to be called from a vlib/vnet app */
766 vl_mem_api_handler_with_vm_node (api_main_t *am, svm_region_t *vlib_rp,
767 void *the_msg, vlib_main_t *vm,
768 vlib_node_runtime_t *node, u8 is_private)
770 u16 id = clib_net_to_host_u16 (*((u16 *) the_msg));
771 vl_api_msg_data_t *m = vl_api_get_msg_data (am, id);
772 u8 *(*handler) (void *, void *, void *);
773 svm_region_t *old_vlib_rp;
774 void *save_shmem_hdr;
777 if (PREDICT_FALSE (am->elog_trace_api_messages))
779 ELOG_TYPE_DECLARE (e) = {
780 .format = "api-msg: %s",
787 ed = ELOG_DATA (am->elog_main, e);
789 ed->c = elog_string (am->elog_main, (char *) m->name);
791 ed->c = elog_string (am->elog_main, "BOGUS");
796 handler = (void *) m->handler;
798 if (PREDICT_FALSE (am->rx_trace && am->rx_trace->enabled))
799 vl_msg_api_trace (am, am->rx_trace, the_msg);
801 if (PREDICT_FALSE (am->msg_print_flag))
803 fformat (stdout, "[%d]: %s\n", id, m->name);
804 fformat (stdout, "%U", format_vl_api_msg_text, am, id, the_msg);
806 is_mp_safe = am->msg_data[id].is_mp_safe;
810 vl_msg_api_barrier_trace_context (am->msg_data[id].name);
811 vl_msg_api_barrier_sync ();
815 old_vlib_rp = am->vlib_rp;
816 save_shmem_hdr = am->shmem_hdr;
817 am->vlib_rp = vlib_rp;
818 am->shmem_hdr = (void *) vlib_rp->user_ctx;
821 if (PREDICT_FALSE (vl_mem_api_fuzz_hook != 0))
822 (*vl_mem_api_fuzz_hook) (id, the_msg);
824 if (m->is_autoendian)
826 void (*endian_fp) (void *);
827 endian_fp = am->msg_data[id].endian_handler;
828 (*endian_fp) (the_msg);
830 if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
831 clib_call_callbacks (am->perf_counter_cbs, am, id, 0 /* before */);
833 (*handler) (the_msg, vm, node);
835 if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
836 clib_call_callbacks (am->perf_counter_cbs, am, id, 1 /* after */);
839 am->vlib_rp = old_vlib_rp;
840 am->shmem_hdr = save_shmem_hdr;
843 vl_msg_api_barrier_release ();
847 clib_warning ("no handler for msg id %d", id);
851 * Special-case, so we can e.g. bounce messages off the vnet
852 * main thread without copying them...
854 if (!m || !m->bounce)
858 old_vlib_rp = am->vlib_rp;
859 save_shmem_hdr = am->shmem_hdr;
860 am->vlib_rp = vlib_rp;
861 am->shmem_hdr = (void *) vlib_rp->user_ctx;
863 vl_msg_api_free (the_msg);
866 am->vlib_rp = old_vlib_rp;
867 am->shmem_hdr = save_shmem_hdr;
871 if (PREDICT_FALSE (am->elog_trace_api_messages))
873 ELOG_TYPE_DECLARE (e) = { .format = "api-msg-done(%s): %s",
874 .format_args = "t4T4",
886 ed = ELOG_DATA (am->elog_main, e);
888 ed->c = elog_string (am->elog_main, (char *) m->name);
890 ed->c = elog_string (am->elog_main, "BOGUS");
891 ed->barrier = is_mp_safe;
896 void_mem_api_handle_msg_i (api_main_t * am, svm_region_t * vlib_rp,
897 vlib_main_t * vm, vlib_node_runtime_t * node,
903 q = ((vl_shmem_hdr_t *) (void *) vlib_rp->user_ctx)->vl_input_queue;
905 if (!svm_queue_sub2 (q, (u8 *) & mp))
907 VL_MSG_API_UNPOISON ((void *) mp);
908 vl_mem_api_handler_with_vm_node (am, vlib_rp, (void *) mp, vm, node,
916 vl_mem_api_handle_msg_main (vlib_main_t * vm, vlib_node_runtime_t * node)
918 api_main_t *am = vlibapi_get_main ();
919 return void_mem_api_handle_msg_i (am, am->vlib_rp, vm, node,
920 0 /* is_private */ );
924 vl_mem_api_handle_rpc (vlib_main_t * vm, vlib_node_runtime_t * node)
926 api_main_t *am = vlibapi_get_main ();
931 * Swap pending and processing vectors, then process the RPCs
932 * Avoid deadlock conditions by construction.
934 clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
935 tmp = vm->processing_rpc_requests;
936 vec_reset_length (tmp);
937 vm->processing_rpc_requests = vm->pending_rpc_requests;
938 vm->pending_rpc_requests = tmp;
939 clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
942 * RPCs are used to reflect function calls to thread 0
943 * when the underlying code is not thread-safe.
945 * Grabbing the thread barrier across a set of RPCs
946 * greatly increases efficiency, and avoids
947 * running afoul of the barrier sync holddown timer.
948 * The barrier sync code supports recursive locking.
950 * We really need to rewrite RPC-based code...
952 if (PREDICT_TRUE (vec_len (vm->processing_rpc_requests)))
954 vl_msg_api_barrier_sync ();
955 for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
957 mp = vm->processing_rpc_requests[i];
958 vl_mem_api_handler_with_vm_node (am, am->vlib_rp, (void *) mp, vm,
959 node, 0 /* is_private */);
961 vl_msg_api_barrier_release ();
968 vl_mem_api_handle_msg_private (vlib_main_t * vm, vlib_node_runtime_t * node,
971 api_main_t *am = vlibapi_get_main ();
972 return void_mem_api_handle_msg_i (am, am->vlib_private_rps[reg_index], vm,
973 node, 1 /* is_private */ );
976 vl_api_registration_t *
977 vl_mem_api_client_index_to_registration (u32 handle)
979 vl_api_registration_t **regpp;
980 vl_api_registration_t *regp;
981 api_main_t *am = vlibapi_get_main ();
982 vl_shmem_hdr_t *shmem_hdr;
985 index = vl_msg_api_handle_get_index (handle);
986 regpp = am->vl_clients + index;
988 if (pool_is_free (am->vl_clients, regpp))
990 vl_msg_api_increment_missing_client_counter ();
995 shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
996 if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
998 vl_msg_api_increment_missing_client_counter ();
1006 vl_api_client_index_to_input_queue (u32 index)
1008 vl_api_registration_t *regp;
1009 api_main_t *am = vlibapi_get_main ();
1011 /* Special case: vlib trying to send itself a message */
1012 if (index == (u32) ~ 0)
1013 return (am->shmem_hdr->vl_input_queue);
1015 regp = vl_mem_api_client_index_to_registration (index);
1018 return (regp->vl_input_queue);
1021 static clib_error_t *
1022 setup_memclnt_exit (vlib_main_t * vm)
1024 atexit (vl_unmap_shmem);
1028 VLIB_INIT_FUNCTION (setup_memclnt_exit);
1031 format_api_message_rings (u8 * s, va_list * args)
1033 api_main_t *am = va_arg (*args, api_main_t *);
1034 vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
1035 int main_segment = va_arg (*args, int);
1040 return format (s, "%8s %8s %8s %8s %8s\n",
1041 "Owner", "Size", "Nitems", "Hits", "Misses");
1043 ap = shmem_hdr->vl_rings;
1045 for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
1047 s = format (s, "%8s %8d %8d %8d %8d\n",
1048 "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
1052 ap = shmem_hdr->client_rings;
1054 for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
1056 s = format (s, "%8s %8d %8d %8d %8d\n",
1057 "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
1063 s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
1066 "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
1067 shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
1068 shmem_hdr->garbage_collects);
1073 static clib_error_t *
1074 vl_api_ring_command (vlib_main_t * vm,
1075 unformat_input_t * input, vlib_cli_command_t * cli_cmd)
1078 vl_shmem_hdr_t *shmem_hdr;
1079 api_main_t *am = vlibapi_get_main ();
1081 /* First, dump the primary region rings.. */
1083 if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
1085 vlib_cli_output (vm, "Shared memory segment not initialized...\n");
1089 shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
1091 vlib_cli_output (vm, "Main API segment rings:");
1093 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1094 0 /* print header */ , 0 /* notused */ );
1096 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1097 shmem_hdr, 1 /* main segment */ );
1099 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
1101 svm_region_t *vlib_rp = am->vlib_private_rps[i];
1102 shmem_hdr = (void *) vlib_rp->user_ctx;
1103 vl_api_registration_t **regpp;
1104 vl_api_registration_t *regp = 0;
1106 /* For horizontal scaling, add a hash table... */
1107 pool_foreach (regpp, am->vl_clients)
1110 if (regp && regp->vlib_rp == vlib_rp)
1112 vlib_cli_output (vm, "%s segment rings:", regp->name);
1116 vlib_cli_output (vm, "regp %llx not found?", regp);
1119 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1120 0 /* print header */ , 0 /* notused */ );
1121 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1122 shmem_hdr, 0 /* main segment */ );
1129 * Display binary api message allocation ring statistics
1131 VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
1133 .path = "show api ring-stats",
1134 .short_help = "Message ring statistics",
1135 .function = vl_api_ring_command,
1139 vlibmemory_init (vlib_main_t * vm)
1141 api_main_t *am = vlibapi_get_main ();
1142 svm_map_region_args_t _a, *a = &_a;
1143 u8 *remove_path1, *remove_path2;
1144 void vlibsocket_reference (void);
1146 vlibsocket_reference ();
1149 * By popular request / to avoid support fires, remove any old api segment
1152 if (am->root_path == 0)
1154 remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
1155 remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
1159 remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
1160 remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
1163 (void) unlink ((char *) remove_path1);
1164 (void) unlink ((char *) remove_path2);
1166 vec_free (remove_path1);
1167 vec_free (remove_path2);
1169 clib_memset (a, 0, sizeof (*a));
1170 a->root_path = am->root_path;
1171 a->name = SVM_GLOBAL_REGION_NAME;
1172 a->baseva = (am->global_baseva != 0) ?
1173 am->global_baseva : +svm_get_global_region_base_va ();
1174 a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
1175 a->flags = SVM_FLAGS_NODATA;
1176 a->uid = am->api_uid;
1177 a->gid = am->api_gid;
1179 (am->global_pvt_heap_size !=
1180 0) ? am->global_pvt_heap_size : SVM_PVT_MHEAP_SIZE;
1182 svm_region_init_args (a);
1188 vl_set_memory_region_name (const char *name)
1190 api_main_t *am = vlibapi_get_main ();
1191 am->region_name = name;
1195 * fd.io coding-style-patch-verification: ON
1198 * eval: (c-set-style "gnu")