2 *------------------------------------------------------------------
3 * Copyright (c) 2018 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
19 #include <vlib/vlib.h>
20 #include <vlibapi/api.h>
21 #include <vlibmemory/api.h>
22 #include <vlibmemory/memory_api.h>
24 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
26 #define vl_typedefs /* define message structures */
27 #include <vlibmemory/vl_memory_api_h.h>
30 /* instantiate all the print functions we know about */
32 #include <vlibmemory/vl_memory_api_h.h>
35 /* instantiate all the endian swap functions we know about */
37 #include <vlibmemory/vl_memory_api_h.h>
40 volatile int **vl_api_queue_cursizes;
43 memclnt_queue_callback (vlib_main_t * vm)
46 api_main_t *am = vlibapi_get_main ();
47 int have_pending_rpcs;
49 if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) !=
50 1 + vec_len (am->vlib_private_rps)))
52 vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
58 q = shmem_hdr->vl_input_queue;
62 vec_add1 (vl_api_queue_cursizes, &q->cursize);
64 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
66 svm_region_t *vlib_rp = am->vlib_private_rps[i];
68 shmem_hdr = (void *) vlib_rp->user_ctx;
69 q = shmem_hdr->vl_input_queue;
70 vec_add1 (vl_api_queue_cursizes, &q->cursize);
74 for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
76 if (*vl_api_queue_cursizes[i])
78 vm->queue_signal_pending = 1;
79 vm->api_queue_nonempty = 1;
80 vlib_process_signal_event (vm, vl_api_clnt_node.index,
81 /* event_type */ QUEUE_SIGNAL_EVENT,
87 clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
88 have_pending_rpcs = vec_len (vm->pending_rpc_requests) > 0;
89 clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
91 if (have_pending_rpcs)
93 vm->queue_signal_pending = 1;
94 vm->api_queue_nonempty = 1;
95 vlib_process_signal_event (vm, vl_api_clnt_node.index,
96 /* event_type */ QUEUE_SIGNAL_EVENT,
102 * vl_api_memclnt_create_internal
105 vl_api_memclnt_create_internal (char *name, svm_queue_t * q)
107 vl_api_registration_t **regpp;
108 vl_api_registration_t *regp;
110 api_main_t *am = vlibapi_get_main ();
112 ASSERT (vlib_get_thread_index () == 0);
113 pool_get (am->vl_clients, regpp);
116 oldheap = vl_msg_push_heap ();
117 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
120 clib_memset (regp, 0, sizeof (*regp));
121 regp->registration_type = REGISTRATION_TYPE_SHMEM;
122 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
123 regp->vlib_rp = am->vlib_rp;
124 regp->shmem_hdr = am->shmem_hdr;
126 regp->vl_input_queue = q;
127 regp->name = format (0, "%s%c", name, 0);
129 vl_msg_pop_heap (oldheap);
130 return vl_msg_api_handle_from_index_and_epoch
131 (regp->vl_api_registration_pool_index,
132 am->shmem_hdr->application_restarts);
136 * vl_api_memclnt_create_t_handler
139 vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp)
141 vl_api_registration_t **regpp;
142 vl_api_registration_t *regp;
143 vl_api_memclnt_create_reply_t *rp;
147 api_main_t *am = vlibapi_get_main ();
151 * This is tortured. Maintain a vlib-address-space private
152 * pool of client registrations. We use the shared-memory virtual
153 * address of client structure as a handle, to allow direct
154 * manipulation of context quota vbls from the client library.
156 * This scheme causes trouble w/ API message trace replay, since
157 * some random VA from clib_mem_alloc() certainly won't
158 * occur in the Linux sim. The (very) few places
159 * that care need to use the pool index.
161 * Putting the registration object(s) into a pool in shared memory and
162 * using the pool index as a handle seems like a great idea.
163 * Unfortunately, each and every reference to that pool would need
164 * to be protected by a mutex:
168 * convert pool index to
176 pool_get (am->vl_clients, regpp);
178 oldheap = vl_msg_push_heap ();
179 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
182 clib_memset (regp, 0, sizeof (*regp));
183 regp->registration_type = REGISTRATION_TYPE_SHMEM;
184 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
185 regp->vlib_rp = am->vlib_rp;
186 regp->shmem_hdr = am->shmem_hdr;
187 regp->clib_file_index = am->shmem_hdr->clib_file_index;
189 q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
190 VL_MSG_API_SVM_QUEUE_UNPOISON (q);
192 regp->name = format (0, "%s", mp->name);
193 vec_add1 (regp->name, 0);
194 regp->keepalive = true;
196 if (am->serialized_message_table_in_shmem == 0)
197 am->serialized_message_table_in_shmem =
198 vl_api_serialize_message_table (am, 0);
200 if (am->vlib_rp != am->vlib_primary_rp)
201 msg_table = vl_api_serialize_message_table (am, 0);
203 msg_table = am->serialized_message_table_in_shmem;
205 vl_msg_pop_heap (oldheap);
207 rp = vl_msg_api_alloc (sizeof (*rp));
208 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
209 rp->handle = (uword) regp;
210 rp->index = vl_msg_api_handle_from_index_and_epoch
211 (regp->vl_api_registration_pool_index,
212 am->shmem_hdr->application_restarts);
213 rp->context = mp->context;
214 rp->response = ntohl (rv);
215 rp->message_table = pointer_to_uword (msg_table);
217 vl_msg_api_send_shmem (q, (u8 *) & rp);
221 vl_api_memclnt_create_v2_t_handler (vl_api_memclnt_create_v2_t *mp)
223 vl_api_registration_t **regpp;
224 vl_api_registration_t *regp;
225 vl_api_memclnt_create_v2_reply_t *rp;
229 api_main_t *am = vlibapi_get_main ();
233 * This is tortured. Maintain a vlib-address-space private
234 * pool of client registrations. We use the shared-memory virtual
235 * address of client structure as a handle, to allow direct
236 * manipulation of context quota vbls from the client library.
238 * This scheme causes trouble w/ API message trace replay, since
239 * some random VA from clib_mem_alloc() certainly won't
240 * occur in the Linux sim. The (very) few places
241 * that care need to use the pool index.
243 * Putting the registration object(s) into a pool in shared memory and
244 * using the pool index as a handle seems like a great idea.
245 * Unfortunately, each and every reference to that pool would need
246 * to be protected by a mutex:
250 * convert pool index to
258 pool_get (am->vl_clients, regpp);
260 oldheap = vl_msg_push_heap ();
261 *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
264 clib_memset (regp, 0, sizeof (*regp));
265 regp->registration_type = REGISTRATION_TYPE_SHMEM;
266 regp->vl_api_registration_pool_index = regpp - am->vl_clients;
267 regp->vlib_rp = am->vlib_rp;
268 regp->shmem_hdr = am->shmem_hdr;
269 regp->clib_file_index = am->shmem_hdr->clib_file_index;
271 q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
272 VL_MSG_API_SVM_QUEUE_UNPOISON (q);
274 regp->name = format (0, "%s", mp->name);
275 vec_add1 (regp->name, 0);
276 regp->keepalive = mp->keepalive;
278 if (am->serialized_message_table_in_shmem == 0)
279 am->serialized_message_table_in_shmem =
280 vl_api_serialize_message_table (am, 0);
282 if (am->vlib_rp != am->vlib_primary_rp)
283 msg_table = vl_api_serialize_message_table (am, 0);
285 msg_table = am->serialized_message_table_in_shmem;
287 vl_msg_pop_heap (oldheap);
289 rp = vl_msg_api_alloc (sizeof (*rp));
290 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_V2_REPLY);
291 rp->handle = (uword) regp;
292 rp->index = vl_msg_api_handle_from_index_and_epoch (
293 regp->vl_api_registration_pool_index, am->shmem_hdr->application_restarts);
294 rp->context = mp->context;
295 rp->response = ntohl (rv);
296 rp->message_table = pointer_to_uword (msg_table);
298 vl_msg_api_send_shmem (q, (u8 *) &rp);
302 vl_api_call_reaper_functions (u32 client_index)
304 clib_error_t *error = 0;
305 _vl_msg_api_function_list_elt_t *i;
307 i = vlibapi_get_main ()->reaper_function_registrations;
310 error = i->f (client_index);
312 clib_error_report (error);
313 i = i->next_init_function;
318 * vl_api_memclnt_delete_t_handler
321 vl_api_memclnt_delete_t_handler (vl_api_memclnt_delete_t * mp)
323 vl_api_registration_t **regpp;
324 vl_api_registration_t *regp;
325 vl_api_memclnt_delete_reply_t *rp;
327 api_main_t *am = vlibapi_get_main ();
328 u32 handle, client_index, epoch;
332 vl_api_call_reaper_functions (handle);
334 epoch = vl_msg_api_handle_get_epoch (handle);
335 client_index = vl_msg_api_handle_get_index (handle);
337 if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
340 ("Stale clnt delete index %d old epoch %d cur epoch %d",
342 (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK));
346 regpp = pool_elt_at_index (am->vl_clients, client_index);
348 if (!pool_is_free (am->vl_clients, regpp))
352 int private_registration = 0;
354 /* Send reply unless client asked us to do the cleanup */
358 * Note: the API message handling path will set am->vlib_rp
359 * as appropriate for pairwise / private memory segments
361 rp = vl_msg_api_alloc (sizeof (*rp));
362 rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
363 rp->handle = mp->handle;
366 vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
367 if (client_index != regp->vl_api_registration_pool_index)
369 clib_warning ("mismatch client_index %d pool_index %d",
371 regp->vl_api_registration_pool_index);
372 vl_msg_api_free (rp);
377 /* No dangling references, please */
380 /* For horizontal scaling, add a hash table... */
381 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
383 /* Is this a pairwise / private API segment? */
384 if (am->vlib_private_rps[i] == am->vlib_rp)
386 /* Note: account for the memfd header page */
387 uword virtual_base = am->vlib_rp->virtual_base - MMAP_PAGESIZE;
388 uword virtual_size = am->vlib_rp->virtual_size + MMAP_PAGESIZE;
391 * Kill the registration pool element before we make
392 * the index vanish forever
394 pool_put_index (am->vl_clients,
395 regp->vl_api_registration_pool_index);
397 vec_delete (am->vlib_private_rps, 1, i);
398 /* Kill it, accounting for the memfd header page */
399 if (munmap ((void *) virtual_base, virtual_size) < 0)
400 clib_unix_warning ("munmap");
401 /* Reset the queue-length-address cache */
402 vec_reset_length (vl_api_queue_cursizes);
403 private_registration = 1;
408 if (private_registration == 0)
410 pool_put_index (am->vl_clients,
411 regp->vl_api_registration_pool_index);
412 oldheap = vl_msg_push_heap ();
414 svm_queue_free (regp->vl_input_queue);
415 vec_free (regp->name);
416 /* Poison the old registration */
417 clib_memset (regp, 0xF1, sizeof (*regp));
418 clib_mem_free (regp);
419 vl_msg_pop_heap (oldheap);
421 * These messages must be freed manually, since they're set up
422 * as "bounce" messages. In the private_registration == 1 case,
423 * we kill the shared-memory segment which contains the message
426 vl_msg_api_free (mp);
431 clib_warning ("unknown client ID %d", mp->index);
436 * client answered a ping, stave off the grim reaper...
439 vl_api_memclnt_keepalive_reply_t_handler
440 (vl_api_memclnt_keepalive_reply_t * mp)
442 vl_api_registration_t *regp;
443 vlib_main_t *vm = vlib_get_main ();
445 regp = vl_api_client_index_to_registration (mp->context);
448 regp->last_heard = vlib_time_now (vm);
449 regp->unanswered_pings = 0;
452 clib_warning ("BUG: anonymous memclnt_keepalive_reply");
456 * We can send ourselves these messages if someone uses the
457 * builtin binary api test tool...
460 vl_api_memclnt_keepalive_t_handler (vl_api_memclnt_keepalive_t * mp)
462 vl_api_memclnt_keepalive_reply_t *rmp;
464 vl_shmem_hdr_t *shmem_hdr;
466 am = vlibapi_get_main ();
467 shmem_hdr = am->shmem_hdr;
469 rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
470 clib_memset (rmp, 0, sizeof (*rmp));
471 rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
472 rmp->context = mp->context;
473 vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
477 * To avoid filling the API trace buffer with boring messages,
478 * don't trace memclnt_keepalive[_reply] msgs
481 #define foreach_vlib_api_msg \
482 _ (MEMCLNT_CREATE, memclnt_create, 0) \
483 _ (MEMCLNT_CREATE_V2, memclnt_create_v2, 0) \
484 _ (MEMCLNT_DELETE, memclnt_delete, 0) \
485 _ (MEMCLNT_KEEPALIVE, memclnt_keepalive, 0) \
486 _ (MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
492 vl_mem_api_init (const char *region_name)
495 api_main_t *am = vlibapi_get_main ();
496 vl_msg_api_msg_config_t cfg;
497 vl_msg_api_msg_config_t *c = &cfg;
499 vlib_main_t *vm = vlib_get_main ();
501 clib_memset (c, 0, sizeof (*c));
503 if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
509 c->id = VL_API_##N; \
511 c->handler = vl_api_##n##_t_handler; \
512 c->endian = vl_api_##n##_t_endian; \
513 c->format_fn = vl_api_##n##_t_format; \
514 c->size = sizeof (vl_api_##n##_t); \
515 c->traced = t; /* trace, so these msgs print */ \
516 c->replay = 0; /* don't replay client create/delete msgs */ \
517 c->message_bounce = 0; /* don't bounce this message */ \
518 vl_msg_api_config (c); \
522 foreach_vlib_api_msg;
525 #define vl_msg_name_crc_list
526 #include <vlibmemory/memclnt.api.h>
527 #undef vl_msg_name_crc_list
529 #define _(id, n, crc) vl_msg_api_add_msg_name_crc (am, #n "_" #crc, id);
530 foreach_vl_msg_name_crc_memclnt;
534 * special-case freeing of memclnt_delete messages, so we can
535 * simply munmap pairwise / private API segments...
537 am->msg_data[VL_API_MEMCLNT_DELETE].bounce = 1;
538 vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE_REPLY, 1);
539 vl_api_set_msg_thread_safe (am, VL_API_MEMCLNT_KEEPALIVE, 1);
541 vlib_set_queue_signal_callback (vm, memclnt_queue_callback);
544 ASSERT (shm && shm->vl_input_queue);
546 /* Make a note so we can always find the primary region easily */
547 am->vlib_primary_rp = am->vlib_rp;
553 map_api_segment_init (vlib_main_t * vm)
555 api_main_t *am = vlibapi_get_main ();
558 if ((rv = vl_mem_api_init (am->region_name)) < 0)
560 return clib_error_return (0, "vl_mem_api_init (%s) failed",
567 send_memclnt_keepalive (vl_api_registration_t * regp, f64 now)
569 vl_api_memclnt_keepalive_t *mp;
571 api_main_t *am = vlibapi_get_main ();
573 q = regp->vl_input_queue;
576 * If the queue head is moving, assume that the client is processing
577 * messages and skip the ping. This heuristic may fail if the queue
578 * is in the same position as last time, net of wrapping; in which
579 * case, the client will receive a keepalive.
581 if (regp->last_queue_head != q->head)
583 regp->last_heard = now;
584 regp->unanswered_pings = 0;
585 regp->last_queue_head = q->head;
590 * push/pop shared memory segment, so this routine
591 * will work with "normal" as well as "private segment"
595 mp = vl_mem_api_alloc_as_if_client_w_reg (regp, sizeof (*mp));
596 clib_memset (mp, 0, sizeof (*mp));
597 mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
598 mp->context = mp->client_index =
599 vl_msg_api_handle_from_index_and_epoch
600 (regp->vl_api_registration_pool_index,
601 am->shmem_hdr->application_restarts);
603 regp->unanswered_pings++;
605 /* Failure-to-send due to a stuffed queue is absolutely expected */
606 if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
607 vl_msg_api_free_w_region (regp->vlib_rp, mp);
611 vl_mem_send_client_keepalive_w_reg (api_main_t * am, f64 now,
612 vl_api_registration_t ** regpp,
614 u32 ** confused_indices)
616 vl_api_registration_t *regp = *regpp;
619 /* If we haven't heard from this client recently... */
620 if (regp->last_heard < (now - 10.0))
622 if (regp->unanswered_pings == 2)
625 q = regp->vl_input_queue;
626 if (kill (q->consumer_pid, 0) >= 0)
628 clib_warning ("REAPER: lazy binary API client '%s'",
630 regp->unanswered_pings = 0;
631 regp->last_heard = now;
635 clib_warning ("REAPER: binary API client '%s' died",
637 vec_add1 (*dead_indices, regpp - am->vl_clients);
641 send_memclnt_keepalive (regp, now);
644 regp->unanswered_pings = 0;
648 clib_warning ("NULL client registration index %d",
649 regpp - am->vl_clients);
650 vec_add1 (*confused_indices, regpp - am->vl_clients);
655 vl_mem_api_dead_client_scan (api_main_t * am, vl_shmem_hdr_t * shm, f64 now)
657 vl_api_registration_t **regpp;
658 static u32 *dead_indices;
659 static u32 *confused_indices;
661 vec_reset_length (dead_indices);
662 vec_reset_length (confused_indices);
665 pool_foreach (regpp, am->vl_clients) {
666 if (!(*regpp)->keepalive)
668 vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
673 /* This should "never happen," but if it does, fix it... */
674 if (PREDICT_FALSE (vec_len (confused_indices) > 0))
677 for (i = 0; i < vec_len (confused_indices); i++)
679 pool_put_index (am->vl_clients, confused_indices[i]);
683 if (PREDICT_FALSE (vec_len (dead_indices) > 0))
688 /* Allow the application to clean up its registrations */
689 for (i = 0; i < vec_len (dead_indices); i++)
691 regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
696 handle = vl_msg_api_handle_from_index_and_epoch
697 (dead_indices[i], shm->application_restarts);
698 vl_api_call_reaper_functions (handle);
702 oldheap = vl_msg_push_heap ();
704 for (i = 0; i < vec_len (dead_indices); i++)
706 regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
709 /* Is this a pairwise SVM segment? */
710 if ((*regpp)->vlib_rp != am->vlib_rp)
713 svm_region_t *dead_rp = (*regpp)->vlib_rp;
714 /* Note: account for the memfd header page */
715 uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
716 uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
718 /* For horizontal scaling, add a hash table... */
719 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
720 if (am->vlib_private_rps[i] == dead_rp)
722 vec_delete (am->vlib_private_rps, 1, i);
725 svm_pop_heap (oldheap);
726 clib_warning ("private rp %llx AWOL", dead_rp);
727 oldheap = svm_push_data_heap (am->vlib_rp);
730 /* Kill it, accounting for the memfd header page */
731 svm_pop_heap (oldheap);
732 if (munmap ((void *) virtual_base, virtual_size) < 0)
733 clib_unix_warning ("munmap");
734 /* Reset the queue-length-address cache */
735 vec_reset_length (vl_api_queue_cursizes);
736 oldheap = svm_push_data_heap (am->vlib_rp);
740 /* Poison the old registration */
741 clib_memset (*regpp, 0xF3, sizeof (**regpp));
742 clib_mem_free (*regpp);
744 /* no dangling references, please */
749 svm_pop_heap (oldheap);
750 clib_warning ("Duplicate free, client index %d",
751 regpp - am->vl_clients);
752 oldheap = svm_push_data_heap (am->vlib_rp);
756 svm_client_scan_this_region_nolock (am->vlib_rp);
758 vl_msg_pop_heap (oldheap);
759 for (i = 0; i < vec_len (dead_indices); i++)
760 pool_put_index (am->vl_clients, dead_indices[i]);
764 void (*vl_mem_api_fuzz_hook) (u16, void *);
766 /* This is only to be called from a vlib/vnet app */
768 vl_mem_api_handler_with_vm_node (api_main_t *am, svm_region_t *vlib_rp,
769 void *the_msg, vlib_main_t *vm,
770 vlib_node_runtime_t *node, u8 is_private)
772 u16 id = clib_net_to_host_u16 (*((u16 *) the_msg));
773 vl_api_msg_data_t *m = vl_api_get_msg_data (am, id);
774 u8 *(*handler) (void *, void *, void *);
775 svm_region_t *old_vlib_rp;
776 void *save_shmem_hdr;
779 if (PREDICT_FALSE (am->elog_trace_api_messages))
781 ELOG_TYPE_DECLARE (e) = {
782 .format = "api-msg: %s",
789 ed = ELOG_DATA (am->elog_main, e);
791 ed->c = elog_string (am->elog_main, (char *) m->name);
793 ed->c = elog_string (am->elog_main, "BOGUS");
798 handler = (void *) m->handler;
800 if (PREDICT_FALSE (am->rx_trace && am->rx_trace->enabled))
801 vl_msg_api_trace (am, am->rx_trace, the_msg);
803 if (PREDICT_FALSE (am->msg_print_flag))
805 fformat (stdout, "[%d]: %s\n", id, m->name);
806 fformat (stdout, "%U", format_vl_api_msg_text, am, id, the_msg);
808 is_mp_safe = am->msg_data[id].is_mp_safe;
812 vl_msg_api_barrier_trace_context (am->msg_data[id].name);
813 vl_msg_api_barrier_sync ();
817 old_vlib_rp = am->vlib_rp;
818 save_shmem_hdr = am->shmem_hdr;
819 am->vlib_rp = vlib_rp;
820 am->shmem_hdr = (void *) vlib_rp->user_ctx;
823 if (PREDICT_FALSE (vl_mem_api_fuzz_hook != 0))
824 (*vl_mem_api_fuzz_hook) (id, the_msg);
826 if (m->is_autoendian)
828 void (*endian_fp) (void *);
829 endian_fp = am->msg_data[id].endian_handler;
830 (*endian_fp) (the_msg);
832 if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
833 clib_call_callbacks (am->perf_counter_cbs, am, id, 0 /* before */);
835 (*handler) (the_msg, vm, node);
837 if (PREDICT_FALSE (vec_len (am->perf_counter_cbs) != 0))
838 clib_call_callbacks (am->perf_counter_cbs, am, id, 1 /* after */);
841 am->vlib_rp = old_vlib_rp;
842 am->shmem_hdr = save_shmem_hdr;
845 vl_msg_api_barrier_release ();
849 clib_warning ("no handler for msg id %d", id);
853 * Special-case, so we can e.g. bounce messages off the vnet
854 * main thread without copying them...
856 if (!m || !m->bounce)
860 old_vlib_rp = am->vlib_rp;
861 save_shmem_hdr = am->shmem_hdr;
862 am->vlib_rp = vlib_rp;
863 am->shmem_hdr = (void *) vlib_rp->user_ctx;
865 vl_msg_api_free (the_msg);
868 am->vlib_rp = old_vlib_rp;
869 am->shmem_hdr = save_shmem_hdr;
873 if (PREDICT_FALSE (am->elog_trace_api_messages))
875 ELOG_TYPE_DECLARE (e) = { .format = "api-msg-done(%s): %s",
876 .format_args = "t4T4",
888 ed = ELOG_DATA (am->elog_main, e);
890 ed->c = elog_string (am->elog_main, (char *) m->name);
892 ed->c = elog_string (am->elog_main, "BOGUS");
893 ed->barrier = is_mp_safe;
898 void_mem_api_handle_msg_i (api_main_t * am, svm_region_t * vlib_rp,
899 vlib_main_t * vm, vlib_node_runtime_t * node,
905 q = ((vl_shmem_hdr_t *) (void *) vlib_rp->user_ctx)->vl_input_queue;
907 if (!svm_queue_sub2 (q, (u8 *) & mp))
909 VL_MSG_API_UNPOISON ((void *) mp);
910 vl_mem_api_handler_with_vm_node (am, vlib_rp, (void *) mp, vm, node,
918 vl_mem_api_handle_msg_main (vlib_main_t * vm, vlib_node_runtime_t * node)
920 api_main_t *am = vlibapi_get_main ();
921 return void_mem_api_handle_msg_i (am, am->vlib_rp, vm, node,
922 0 /* is_private */ );
926 vl_mem_api_handle_rpc (vlib_main_t * vm, vlib_node_runtime_t * node)
928 api_main_t *am = vlibapi_get_main ();
933 * Swap pending and processing vectors, then process the RPCs
934 * Avoid deadlock conditions by construction.
936 clib_spinlock_lock_if_init (&vm->pending_rpc_lock);
937 tmp = vm->processing_rpc_requests;
938 vec_reset_length (tmp);
939 vm->processing_rpc_requests = vm->pending_rpc_requests;
940 vm->pending_rpc_requests = tmp;
941 clib_spinlock_unlock_if_init (&vm->pending_rpc_lock);
944 * RPCs are used to reflect function calls to thread 0
945 * when the underlying code is not thread-safe.
947 * Grabbing the thread barrier across a set of RPCs
948 * greatly increases efficiency, and avoids
949 * running afoul of the barrier sync holddown timer.
950 * The barrier sync code supports recursive locking.
952 * We really need to rewrite RPC-based code...
954 if (PREDICT_TRUE (vec_len (vm->processing_rpc_requests)))
956 vl_msg_api_barrier_sync ();
957 for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
959 mp = vm->processing_rpc_requests[i];
960 vl_mem_api_handler_with_vm_node (am, am->vlib_rp, (void *) mp, vm,
961 node, 0 /* is_private */);
963 vl_msg_api_barrier_release ();
970 vl_mem_api_handle_msg_private (vlib_main_t * vm, vlib_node_runtime_t * node,
973 api_main_t *am = vlibapi_get_main ();
974 return void_mem_api_handle_msg_i (am, am->vlib_private_rps[reg_index], vm,
975 node, 1 /* is_private */ );
978 vl_api_registration_t *
979 vl_mem_api_client_index_to_registration (u32 handle)
981 vl_api_registration_t **regpp;
982 vl_api_registration_t *regp;
983 api_main_t *am = vlibapi_get_main ();
984 vl_shmem_hdr_t *shmem_hdr;
987 index = vl_msg_api_handle_get_index (handle);
988 regpp = am->vl_clients + index;
990 if (pool_is_free (am->vl_clients, regpp))
992 vl_msg_api_increment_missing_client_counter ();
997 shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
998 if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
1000 vl_msg_api_increment_missing_client_counter ();
1008 vl_api_client_index_to_input_queue (u32 index)
1010 vl_api_registration_t *regp;
1011 api_main_t *am = vlibapi_get_main ();
1013 /* Special case: vlib trying to send itself a message */
1014 if (index == (u32) ~ 0)
1015 return (am->shmem_hdr->vl_input_queue);
1017 regp = vl_mem_api_client_index_to_registration (index);
1020 return (regp->vl_input_queue);
1023 static clib_error_t *
1024 setup_memclnt_exit (vlib_main_t * vm)
1026 atexit (vl_unmap_shmem);
1030 VLIB_INIT_FUNCTION (setup_memclnt_exit);
1033 format_api_message_rings (u8 * s, va_list * args)
1035 api_main_t *am = va_arg (*args, api_main_t *);
1036 vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
1037 int main_segment = va_arg (*args, int);
1042 return format (s, "%8s %8s %8s %8s %8s\n",
1043 "Owner", "Size", "Nitems", "Hits", "Misses");
1045 ap = shmem_hdr->vl_rings;
1047 for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
1049 s = format (s, "%8s %8d %8d %8d %8d\n",
1050 "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
1054 ap = shmem_hdr->client_rings;
1056 for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
1058 s = format (s, "%8s %8d %8d %8d %8d\n",
1059 "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
1065 s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
1068 "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
1069 shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
1070 shmem_hdr->garbage_collects);
1075 static clib_error_t *
1076 vl_api_ring_command (vlib_main_t * vm,
1077 unformat_input_t * input, vlib_cli_command_t * cli_cmd)
1080 vl_shmem_hdr_t *shmem_hdr;
1081 api_main_t *am = vlibapi_get_main ();
1083 /* First, dump the primary region rings.. */
1085 if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
1087 vlib_cli_output (vm, "Shared memory segment not initialized...\n");
1091 shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
1093 vlib_cli_output (vm, "Main API segment rings:");
1095 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1096 0 /* print header */ , 0 /* notused */ );
1098 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1099 shmem_hdr, 1 /* main segment */ );
1101 for (i = 0; i < vec_len (am->vlib_private_rps); i++)
1103 svm_region_t *vlib_rp = am->vlib_private_rps[i];
1104 shmem_hdr = (void *) vlib_rp->user_ctx;
1105 vl_api_registration_t **regpp;
1106 vl_api_registration_t *regp = 0;
1108 /* For horizontal scaling, add a hash table... */
1110 pool_foreach (regpp, am->vl_clients)
1113 if (regp && regp->vlib_rp == vlib_rp)
1115 vlib_cli_output (vm, "%s segment rings:", regp->name);
1119 vlib_cli_output (vm, "regp %llx not found?", regp);
1123 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1124 0 /* print header */ , 0 /* notused */ );
1125 vlib_cli_output (vm, "%U", format_api_message_rings, am,
1126 shmem_hdr, 0 /* main segment */ );
1133 * Display binary api message allocation ring statistics
1136 VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
1138 .path = "show api ring-stats",
1139 .short_help = "Message ring statistics",
1140 .function = vl_api_ring_command,
1145 vlibmemory_init (vlib_main_t * vm)
1147 api_main_t *am = vlibapi_get_main ();
1148 svm_map_region_args_t _a, *a = &_a;
1149 u8 *remove_path1, *remove_path2;
1150 void vlibsocket_reference (void);
1152 vlibsocket_reference ();
1155 * By popular request / to avoid support fires, remove any old api segment
1158 if (am->root_path == 0)
1160 remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
1161 remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
1165 remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
1166 remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
1169 (void) unlink ((char *) remove_path1);
1170 (void) unlink ((char *) remove_path2);
1172 vec_free (remove_path1);
1173 vec_free (remove_path2);
1175 clib_memset (a, 0, sizeof (*a));
1176 a->root_path = am->root_path;
1177 a->name = SVM_GLOBAL_REGION_NAME;
1178 a->baseva = (am->global_baseva != 0) ?
1179 am->global_baseva : +svm_get_global_region_base_va ();
1180 a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
1181 a->flags = SVM_FLAGS_NODATA;
1182 a->uid = am->api_uid;
1183 a->gid = am->api_gid;
1185 (am->global_pvt_heap_size !=
1186 0) ? am->global_pvt_heap_size : SVM_PVT_MHEAP_SIZE;
1188 svm_region_init_args (a);
1194 vl_set_memory_region_name (const char *name)
1196 api_main_t *am = vlibapi_get_main ();
1197 am->region_name = name;
1201 * fd.io coding-style-patch-verification: ON
1204 * eval: (c-set-style "gnu")