2 * Copyright (c) 2020 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #ifndef SRC_SVM_FIFO_TYPES_H_
17 #define SRC_SVM_FIFO_TYPES_H_
20 #include <vppinfra/clib.h>
21 #include <vppinfra/rbtree.h>
22 #include <vppinfra/lock.h>
24 #define FS_MIN_LOG2_CHUNK_SZ 12 /**< also min fifo size */
25 #define FS_MAX_LOG2_CHUNK_SZ 22 /**< 4MB max chunk size */
26 #define FS_CHUNK_VEC_LEN 11 /**< number of chunk sizes */
28 STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
30 "update chunk sizes");
32 #define SVM_FIFO_TRACE (0)
33 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7
35 typedef struct fifo_segment_header_ fifo_segment_header_t;
36 typedef uword fs_sptr_t;
38 typedef struct svm_fifo_chunk_
40 u32 start_byte; /**< chunk start byte */
41 u32 length; /**< length of chunk in bytes */
42 fs_sptr_t next; /**< pointer to next chunk in linked-lists */
43 rb_node_index_t enq_rb_index; /**< enq node index if chunk in rbtree */
44 rb_node_index_t deq_rb_index; /**< deq node index if chunk in rbtree */
45 u8 data[0]; /**< start of chunk data */
50 u32 next; /**< Next linked-list element pool index */
51 u32 prev; /**< Previous linked-list element pool index */
52 u32 start; /**< Start of segment, normalized*/
53 u32 length; /**< Length of segment */
61 } svm_fifo_trace_elem_t;
63 typedef struct svm_fifo_shr_
65 CLIB_CACHE_LINE_ALIGN_MARK (shared);
66 fs_sptr_t start_chunk; /**< first chunk in fifo chunk list */
67 fs_sptr_t end_chunk; /**< end chunk in fifo chunk list */
68 volatile u32 has_event; /**< non-zero if deq event exists */
69 u32 min_alloc; /**< min chunk alloc if space available */
70 u32 size; /**< size of the fifo in bytes */
71 u32 master_session_index; /**< session layer session index */
72 u32 client_session_index; /**< app session index */
73 u8 slice_index; /**< segment slice for fifo */
74 fs_sptr_t next; /**< next in freelist */
76 CLIB_CACHE_LINE_ALIGN_MARK (consumer);
77 fs_sptr_t head_chunk; /**< tracks chunk where head lands */
78 u32 head; /**< fifo head position/byte */
79 volatile u32 want_deq_ntf; /**< producer wants nudge */
80 volatile u32 has_deq_ntf;
82 CLIB_CACHE_LINE_ALIGN_MARK (producer);
83 u32 tail; /**< fifo tail position/byte */
84 fs_sptr_t tail_chunk; /**< tracks chunk where tail lands */
85 volatile u8 n_subscribers; /**< Number of subscribers for io events */
86 u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
89 typedef struct _svm_fifo
91 CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
92 svm_fifo_shared_t *shr; /**< shared fifo in fifo segment memory */
93 fifo_segment_header_t *fs_hdr; /**< fifo segment header for fifo */
94 rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */
95 rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */
96 svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
97 svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
98 ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
99 u32 ooos_list_head; /**< Head of out-of-order linked-list */
100 u32 ooos_newest; /**< Last segment to have been updated */
102 u8 flags; /**< fifo flags */
103 u8 master_thread_index; /**< session layer thread index */
104 u8 client_thread_index; /**< app worker index */
105 i8 refcnt; /**< reference count */
106 u32 segment_manager; /**< session layer segment manager index */
107 u32 segment_index; /**< segment index in segment manager */
109 struct _svm_fifo *next; /**< prev in active chain */
110 struct _svm_fifo *prev; /**< prev in active chain */
112 svm_fifo_chunk_t *chunks_at_attach; /**< chunks to be accounted at detach */
113 svm_fifo_shared_t *hdr_at_attach; /**< hdr to be freed at detach */
116 svm_fifo_trace_elem_t *trace;
120 typedef struct fifo_segment_slice_
122 CLIB_CACHE_LINE_ALIGN_MARK (cacheline);
123 fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
124 fs_sptr_t free_fifos; /**< Freelists of fifo shared hdrs */
125 uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */
126 uword virtual_mem; /**< Slice sum of all fifo sizes */
127 u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */
128 } fifo_segment_slice_t;
130 typedef struct fifo_slice_private_
132 clib_mem_bulk_handle_t fifos; /**< Bulk fifo allocator */
133 uword virtual_mem; /**< Slice sum of all fifo sizes */
134 svm_fifo_t *active_fifos; /**< Linked list of active RX fifos */
135 } fifo_slice_private_t;
137 struct fifo_segment_header_
139 uword n_cached_bytes; /**< Cached bytes */
140 u32 n_active_fifos; /**< Number of active fifos */
141 u32 n_reserved_bytes; /**< Bytes not to be allocated */
142 u32 max_log2_fifo_size; /**< Max log2(chunk size) for fs */
143 u8 n_slices; /**< Number of slices */
144 u8 pct_first_alloc; /**< Pct of fifo size to alloc */
145 u8 n_mqs; /**< Num mqs for mqs segment */
146 CLIB_CACHE_LINE_ALIGN_MARK (allocator);
148 uword max_byte_index;
149 uword start_byte_index;
150 CLIB_CACHE_LINE_ALIGN_MARK (slice);
151 fifo_segment_slice_t slices[0]; /** Fixed array of slices */
154 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
158 fs_ptr (fifo_segment_header_t *fsh, fs_sptr_t sp)
160 return sp ? (void *) ((u8 *) fsh + sp) : 0;
163 always_inline fs_sptr_t
164 fs_sptr (fifo_segment_header_t *fsh, void *p)
166 return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0;
169 always_inline svm_fifo_chunk_t *
170 fs_chunk_ptr (fifo_segment_header_t *fsh, fs_sptr_t cp)
172 return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0;
175 always_inline fs_sptr_t
176 fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
178 return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0;
181 #endif /* SRC_SVM_FIFO_TYPES_H_ */
184 * fd.io coding-style-patch-verification: ON
187 * eval: (c-set-style "gnu")