X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.h;h=e08b3e9dfa55d4f5be25f132f7de94c327fabe97;hb=0e6199dfa20d4c010bf3ac43e051f9f29c213478;hp=b5b26ac479b369db50533184fc3b0d1788013440;hpb=cefd5d8806b9db7210192f53fdc2b8a60d4dc271;p=vpp.git diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index b5b26ac479b..e08b3e9dfa5 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -23,105 +23,30 @@ #include #include #include -#include +#include -/** Out-of-order segment */ -typedef struct -{ - u32 next; /**< Next linked-list element pool index */ - u32 prev; /**< Previous linked-list element pool index */ - u32 start; /**< Start of segment, normalized*/ - u32 length; /**< Length of segment */ -} ooo_segment_t; - -#define SVM_FIFO_TRACE (0) #define OOO_SEGMENT_INVALID_INDEX ((u32)~0) #define SVM_FIFO_INVALID_SESSION_INDEX ((u32)~0) #define SVM_FIFO_INVALID_INDEX ((u32)~0) -#define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 - -typedef enum svm_fifo_tx_ntf_ -{ - SVM_FIFO_NO_TX_NOTIF = 0, - SVM_FIFO_WANT_TX_NOTIF = 1, - SVM_FIFO_WANT_TX_NOTIF_IF_FULL = 2, -} svm_fifo_tx_ntf_t; - -typedef struct -{ - u32 offset; - u32 len; - u32 action; -} svm_fifo_trace_elem_t; -typedef struct svm_fifo_chunk_ +typedef enum svm_fifo_deq_ntf_ { - u32 start_byte; /**< chunk start byte */ - u32 length; /**< length of chunk in bytes */ - struct svm_fifo_chunk_ *next; /**< pointer to next chunk in linked-lists */ - u8 data[0]; /**< start of chunk data */ -} svm_fifo_chunk_t; + SVM_FIFO_NO_DEQ_NOTIF = 0, /**< No notification requested */ + SVM_FIFO_WANT_DEQ_NOTIF = 1, /**< Notify on dequeue */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL = 2, /**< Notify on transition from full */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY = 4, /**< Notify on transition to empty */ +} svm_fifo_deq_ntf_t; typedef enum svm_fifo_flag_ { - SVM_FIFO_F_MULTI_CHUNK = 1 << 0, - SVM_FIFO_F_GROW = 1 << 1, - SVM_FIFO_F_SHRINK = 1 << 2, - SVM_FIFO_F_COLLECT_CHUNKS = 1 << 3, - SVM_FIFO_F_LL_TRACKED = 1 << 4, + SVM_FIFO_F_LL_TRACKED = 1 << 0, } svm_fifo_flag_t; -typedef struct _svm_fifo -{ - CLIB_CACHE_LINE_ALIGN_MARK (shared_first); - u32 size; /**< size of the fifo in bytes */ - u32 nitems; /**< usable size (size-1) */ - u8 flags; /**< fifo flags */ - svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */ - svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */ - svm_fifo_chunk_t *new_chunks; /**< chunks yet to be added to list */ - rb_tree_t chunk_lookup; - - CLIB_CACHE_LINE_ALIGN_MARK (shared_second); - volatile u32 has_event; /**< non-zero if deq event exists */ - u32 master_session_index; /**< session layer session index */ - u32 client_session_index; /**< app session index */ - u8 master_thread_index; /**< session layer thread index */ - u8 client_thread_index; /**< app worker index */ - i8 refcnt; /**< reference count */ - u32 segment_manager; /**< session layer segment manager index */ - u32 segment_index; /**< segment index in segment manager */ - struct _svm_fifo *next; /**< next in freelist/active chain */ - struct _svm_fifo *prev; /**< prev in active chain */ - u32 size_decrement; /**< bytes to remove from fifo */ - - CLIB_CACHE_LINE_ALIGN_MARK (consumer); - u32 head; /**< fifo head position/byte */ - svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */ - svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ - volatile u32 want_tx_ntf; /**< producer wants nudge */ - volatile u32 has_tx_ntf; - - CLIB_CACHE_LINE_ALIGN_MARK (producer); - u32 tail; /**< fifo tail position/byte */ - u32 ooos_list_head; /**< Head of out-of-order linked-list */ - svm_fifo_chunk_t *tail_chunk; /**< tracks chunk where tail lands */ - svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */ - ooo_segment_t *ooo_segments; /**< Pool of ooo segments */ - u32 ooos_newest; /**< Last segment to have been updated */ - volatile u8 n_subscribers; /**< Number of subscribers for io events */ - u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS]; - -#if SVM_FIFO_TRACE - svm_fifo_trace_elem_t *trace; -#endif - -} svm_fifo_t; - typedef enum { SVM_FIFO_EFULL = -2, SVM_FIFO_EEMPTY = -3, + SVM_FIFO_EGROW = -4, } svm_fifo_err_t; typedef struct svm_fifo_seg_ @@ -188,55 +113,63 @@ f_load_head_tail_all_acq (svm_fifo_t * f, u32 * head, u32 * tail) } /** - * Distance to a from b, i.e., a - b in the fifo + * Fifo current size, i.e., number of bytes enqueued * * Internal function. */ static inline u32 -f_distance_to (svm_fifo_t * f, u32 a, u32 b) +f_cursize (svm_fifo_t * f, u32 head, u32 tail) { - return ((f->size + a - b) % f->size); + return tail - head; } /** - * Distance from a to b, i.e., b - a in the fifo + * Fifo free bytes, i.e., number of free bytes * - * Internal function. + * Internal function */ static inline u32 -f_distance_from (svm_fifo_t * f, u32 a, u32 b) +f_free_count (svm_fifo_t * f, u32 head, u32 tail) { - return ((f->size + b - a) % f->size); + return (f->size - f_cursize (f, head, tail)); } -/** - * Fifo current size, i.e., number of bytes enqueued - * - * Internal function. - */ -static inline u32 -f_cursize (svm_fifo_t * f, u32 head, u32 tail) +always_inline u32 +f_chunk_end (svm_fifo_chunk_t * c) { - return (head <= tail ? tail - head : f->size + tail - head); + return c->start_byte + c->length; } -/** - * Fifo free bytes, i.e., number of free bytes - * - * Internal function - */ -static inline u32 -f_free_count (svm_fifo_t * f, u32 head, u32 tail) +always_inline int +f_pos_lt (u32 a, u32 b) { - return (f->nitems - f_cursize (f, head, tail)); + return ((i32) (a - b) < 0); } -/** - * Try to shrink fifo size. - * - * Internal function. - */ -void svm_fifo_try_shrink (svm_fifo_t * f, u32 head, u32 tail); +always_inline int +f_pos_leq (u32 a, u32 b) +{ + return ((i32) (a - b) <= 0); +} + +always_inline int +f_pos_gt (u32 a, u32 b) +{ + return ((i32) (a - b) > 0); +} + +always_inline int +f_pos_geq (u32 a, u32 b) +{ + return ((i32) (a - b) >= 0); +} + +always_inline u8 +f_chunk_includes_pos (svm_fifo_chunk_t * c, u32 pos) +{ + return (f_pos_geq (pos, c->start_byte) + && f_pos_lt (pos, c->start_byte + c->length)); +} /** * Create fifo of requested size @@ -247,10 +180,11 @@ void svm_fifo_try_shrink (svm_fifo_t * f, u32 head, u32 tail); * rounded to the next highest power-of-two value. * @return pointer to new fifo */ -svm_fifo_t *svm_fifo_create (u32 size); +svm_fifo_t *svm_fifo_alloc (u32 size); /** * Initialize fifo * + * @param f fifo * @param size size for fifo */ void svm_fifo_init (svm_fifo_t * f, u32 size); @@ -266,40 +200,20 @@ void svm_fifo_init (svm_fifo_t * f, u32 size); */ svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size); /** - * Grow fifo size by adding chunk to chunk list + * Ensure the whole fifo size is writeable * - * If fifos are allocated on a segment, this should be called with - * the segment's heap pushed. + * Allocates enough chunks to cover the whole fifo size. * - * @param f fifo to be extended - * @param c chunk or linked list of chunks to be added + * @param f fifo */ -void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c); +int svm_fifo_fill_chunk_list (svm_fifo_t * f); /** - * Request to reduce fifo size by amount of bytes - * - * Because the producer might be enqueuing data when this is called, the - * actual size update is only applied when producer tries to enqueue new - * data, unless @param try_shrink is set. + * Initialize rbtrees used for ooo lookups * * @param f fifo - * @param len number of bytes to remove from fifo. The actual number - * of bytes to be removed will be less or equal to this - * value. - * @param try_shrink flg to indicate if it's safe to try to shrink fifo - * size. It should be set only if this is called by the - * producer of if the producer is not using the fifo - * @return actual length fifo size will be reduced by + * @param ooo_type type of ooo operation (0 enqueue, 1 dequeue) */ -int svm_fifo_reduce_size (svm_fifo_t * f, u32 len, u8 try_shrink); -/** - * Removes chunks that are after fifo end byte - * - * Needs to be called with segment heap pushed. - * - * @param f fifo - */ -svm_fifo_chunk_t *svm_fifo_collect_chunks (svm_fifo_t * f); +void svm_fifo_init_ooo_lookup (svm_fifo_t * f, u8 ooo_type); /** * Free fifo and associated state * @@ -455,13 +369,27 @@ void svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber); * @return number of out of order segments */ u32 svm_fifo_n_ooo_segments (svm_fifo_t * f); -/* +/** * First out-of-order segment for fifo * * @param f fifo * @return first out-of-order segment for fifo */ ooo_segment_t *svm_fifo_first_ooo_segment (svm_fifo_t * f); +/** + * Check if fifo is sane. Debug only. + * + * @param f fifo + * @return 1 if sane, 0 otherwise + */ +u8 svm_fifo_is_sane (svm_fifo_t * f); +/** + * Number of chunks linked into the fifo + * + * @param f fifo + * @return number of chunks in fifo linked list + */ +u32 svm_fifo_n_chunks (svm_fifo_t * f); format_function_t format_svm_fifo; /** @@ -516,7 +444,7 @@ svm_fifo_max_dequeue (svm_fifo_t * f) static inline int svm_fifo_is_full_prod (svm_fifo_t * f) { - return (svm_fifo_max_dequeue_prod (f) == f->nitems); + return (svm_fifo_max_dequeue_prod (f) == f->size); } /* Check if fifo is full. @@ -528,7 +456,7 @@ svm_fifo_is_full_prod (svm_fifo_t * f) static inline int svm_fifo_is_full (svm_fifo_t * f) { - return (svm_fifo_max_dequeue (f) == f->nitems); + return (svm_fifo_max_dequeue (f) == f->size); } /** @@ -595,8 +523,6 @@ svm_fifo_max_enqueue_prod (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_prod (f, &head, &tail); - if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK)) - svm_fifo_try_shrink (f, head, tail); return f_free_count (f, head, tail); } @@ -611,40 +537,68 @@ svm_fifo_max_enqueue (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_all_acq (f, &head, &tail); - if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK)) - svm_fifo_try_shrink (f, head, tail); return f_free_count (f, head, tail); } /** - * Max contiguous chunk of data that can be read + * Max contiguous chunk of data that can be read. + * + * Should only be called by consumers. */ -static inline u32 -svm_fifo_max_read_chunk (svm_fifo_t * f) -{ - u32 head, tail; - f_load_head_tail_cons (f, &head, &tail); - return tail >= head ? (tail - head) : (f->size - head); -} +u32 svm_fifo_max_read_chunk (svm_fifo_t * f); /** * Max contiguous chunk of data that can be written + * + * Should only be called by producers */ -static inline u32 -svm_fifo_max_write_chunk (svm_fifo_t * f) +u32 svm_fifo_max_write_chunk (svm_fifo_t * f); + +/** + * Fifo head chunk getter + * + * @param f fifo + * @return head chunk pointer + */ +static inline svm_fifo_chunk_t * +svm_fifo_head_chunk (svm_fifo_t * f) { - u32 head, tail; - f_load_head_tail_prod (f, &head, &tail); - return tail > head ? f->size - tail : f_free_count (f, head, tail); + return f->head_chunk; } +/** + * Fifo head pointer getter + * + * @param f fifo + * @return head pointer + */ static inline u8 * svm_fifo_head (svm_fifo_t * f) { + if (!f->head_chunk) + return 0; /* load-relaxed: consumer owned index */ return (f->head_chunk->data + (f->head - f->head_chunk->start_byte)); } +/** + * Fifo tail chunk getter + * + * @param f fifo + * @return tail chunk pointer + */ +static inline svm_fifo_chunk_t * +svm_fifo_tail_chunk (svm_fifo_t * f) +{ + return f->tail_chunk; +} + +/** + * Fifo tail pointer getter + * + * @param f fifo + * @return tail pointer + */ static inline u8 * svm_fifo_tail (svm_fifo_t * f) { @@ -652,6 +606,12 @@ svm_fifo_tail (svm_fifo_t * f) return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte)); } +/** + * Fifo number of subscribers getter + * + * @param f fifo + * @return number of subscribers + */ static inline u8 svm_fifo_n_subscribers (svm_fifo_t * f) { @@ -691,7 +651,7 @@ ooo_segment_offset_prod (svm_fifo_t * f, ooo_segment_t * s) /* load-relaxed: producer owned index */ tail = f->tail; - return f_distance_to (f, s->start, tail); + return (s->start - tail); } static inline u32 @@ -700,6 +660,21 @@ ooo_segment_length (svm_fifo_t * f, ooo_segment_t * s) return s->length; } +static inline u32 +svm_fifo_size (svm_fifo_t * f) +{ + return f->size; +} + +static inline void +svm_fifo_set_size (svm_fifo_t * f, u32 size) +{ + if (size > (1 << f->fs_hdr->max_log2_chunk_size)) + return; + fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size); + f->size = size; +} + /** * Check if fifo has io event * @@ -740,95 +715,97 @@ svm_fifo_unset_event (svm_fifo_t * f) } /** - * Set specific want tx notification flag + * Set specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_deq_ntf_t * * @param f fifo * @param ntf_type type of notification requested */ static inline void -svm_fifo_add_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf |= ntf_type; + f->want_deq_ntf |= ntf_type; } /** - * Clear specific want tx notification flag + * Clear specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_ntf_t * * @param f fifo * @param ntf_type type of notification to be cleared */ static inline void -svm_fifo_del_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf &= ~ntf_type; + f->want_deq_ntf &= ~ntf_type; } /** - * Clear the want tx notification flag and set has tx notification + * Clear the want notification flag and set has notification * - * Should be used after enqueuing a tx event. This clears the - * SVM_FIFO_WANT_TX_NOTIF flag but it does not clear - * SVM_FIFO_WANT_TX_NOTIF_IF_FULL. If the latter was set, has_tx_ntf is - * set to avoid enqueueing tx events for for all dequeue operations until + * Should be used after enqueuing an event. This clears the + * SVM_FIFO_WANT_NOTIF flag but it does not clear + * SVM_FIFO_WANT_NOTIF_IF_FULL. If the latter was set, has_ntf is + * set to avoid enqueueing events for for all dequeue operations until * it is manually cleared. * * @param f fifo */ static inline void -svm_fifo_clear_tx_ntf (svm_fifo_t * f) +svm_fifo_clear_deq_ntf (svm_fifo_t * f) { - /* Set the flag if want_tx_notif_if_full was the only ntf requested */ - f->has_tx_ntf = f->want_tx_ntf == SVM_FIFO_WANT_TX_NOTIF_IF_FULL; - svm_fifo_del_want_tx_ntf (f, SVM_FIFO_WANT_TX_NOTIF); + /* Set the flag if want_notif_if_full was the only ntf requested */ + f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; + svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); } /** - * Clear has tx notification flag + * Clear has notification flag * - * The fifo generates only one event per SVM_FIFO_WANT_TX_NOTIF_IF_FULL - * request and sets has_tx_ntf. To received new events the flag must be + * The fifo generates only one event per SVM_FIFO_WANT_NOTIF_IF_FULL + * request and sets has_ntf. To received new events the flag must be * cleared using this function. * * @param f fifo */ static inline void -svm_fifo_reset_tx_ntf (svm_fifo_t * f) +svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) { - f->has_tx_ntf = 0; + f->has_deq_ntf = 0; } /** - * Check if fifo needs tx notification + * Check if fifo needs dequeue notification * - * Determines based on tx notification request flags and state of the fifo if - * a tx io event should be generated. + * Determines based on notification request flags and state of the fifo if + * an event should be generated. * * @param f fifo * @param n_last_deq number of bytes last dequeued - * @return 1 if tx io event should be generated, 0 otherwise + * @return 1 if event should be generated, 0 otherwise */ static inline u8 -svm_fifo_needs_tx_ntf (svm_fifo_t * f, u32 n_last_deq) +svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) { - u8 want_ntf = f->want_tx_ntf; + u8 want_ntf = f->want_deq_ntf; - if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_TX_NOTIF)) + if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)) return 0; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF) + else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF) return 1; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF_IF_FULL) + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) { u32 max_deq = svm_fifo_max_dequeue_cons (f); - u32 nitems = f->nitems; - if (!f->has_tx_ntf && max_deq < nitems - && max_deq + n_last_deq >= nitems) + u32 size = f->size; + if (!f->has_deq_ntf && max_deq < size && max_deq + n_last_deq >= size) + return 1; + } + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) + { + if (!f->has_deq_ntf && svm_fifo_is_empty (f)) return 1; - - return 0; } return 0; }