X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.h;h=2b6e8542cdf120f5edb42d674480442cbcd603d5;hb=refs%2Fchanges%2F74%2F23974%2F34;hp=b5b26ac479b369db50533184fc3b0d1788013440;hpb=cefd5d8806b9db7210192f53fdc2b8a60d4dc271;p=vpp.git diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index b5b26ac479b..2b6e8542cdf 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -40,12 +40,13 @@ typedef struct #define SVM_FIFO_INVALID_INDEX ((u32)~0) #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 -typedef enum svm_fifo_tx_ntf_ +typedef enum svm_fifo_deq_ntf_ { - SVM_FIFO_NO_TX_NOTIF = 0, - SVM_FIFO_WANT_TX_NOTIF = 1, - SVM_FIFO_WANT_TX_NOTIF_IF_FULL = 2, -} svm_fifo_tx_ntf_t; + SVM_FIFO_NO_DEQ_NOTIF = 0, /**< No notification requested */ + SVM_FIFO_WANT_DEQ_NOTIF = 1, /**< Notify on dequeue */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL = 2, /**< Notify on transition from full */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY = 4, /**< Notify on transition to empty */ +} svm_fifo_deq_ntf_t; typedef struct { @@ -59,6 +60,7 @@ typedef struct svm_fifo_chunk_ u32 start_byte; /**< chunk start byte */ u32 length; /**< length of chunk in bytes */ struct svm_fifo_chunk_ *next; /**< pointer to next chunk in linked-lists */ + rb_node_index_t rb_index; /**< node index if chunk in rbtree */ u8 data[0]; /**< start of chunk data */ } svm_fifo_chunk_t; @@ -69,6 +71,7 @@ typedef enum svm_fifo_flag_ SVM_FIFO_F_SHRINK = 1 << 2, SVM_FIFO_F_COLLECT_CHUNKS = 1 << 3, SVM_FIFO_F_LL_TRACKED = 1 << 4, + SVM_FIFO_F_SINGLE_THREAD_OWNED = 1 << 5, } svm_fifo_flag_t; typedef struct _svm_fifo @@ -76,11 +79,12 @@ typedef struct _svm_fifo CLIB_CACHE_LINE_ALIGN_MARK (shared_first); u32 size; /**< size of the fifo in bytes */ u32 nitems; /**< usable size (size-1) */ - u8 flags; /**< fifo flags */ svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */ svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */ - svm_fifo_chunk_t *new_chunks; /**< chunks yet to be added to list */ - rb_tree_t chunk_lookup; + rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */ + rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */ + u8 flags; /**< fifo flags */ + u8 slice_index; /**< segment slice for fifo */ CLIB_CACHE_LINE_ALIGN_MARK (shared_second); volatile u32 has_event; /**< non-zero if deq event exists */ @@ -93,14 +97,15 @@ typedef struct _svm_fifo u32 segment_index; /**< segment index in segment manager */ struct _svm_fifo *next; /**< next in freelist/active chain */ struct _svm_fifo *prev; /**< prev in active chain */ + svm_fifo_chunk_t *new_chunks; /**< chunks yet to be added to list */ u32 size_decrement; /**< bytes to remove from fifo */ CLIB_CACHE_LINE_ALIGN_MARK (consumer); u32 head; /**< fifo head position/byte */ svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */ svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ - volatile u32 want_tx_ntf; /**< producer wants nudge */ - volatile u32 has_tx_ntf; + volatile u32 want_deq_ntf; /**< producer wants nudge */ + volatile u32 has_deq_ntf; CLIB_CACHE_LINE_ALIGN_MARK (producer); u32 tail; /**< fifo tail position/byte */ @@ -251,9 +256,16 @@ svm_fifo_t *svm_fifo_create (u32 size); /** * Initialize fifo * + * @param f fifo * @param size size for fifo */ void svm_fifo_init (svm_fifo_t * f, u32 size); +/** + * Initialize fifo chunks and rbtree + * + * @param f fifo + */ +void svm_fifo_init_chunks (svm_fifo_t * f); /** * Allocate a fifo chunk on heap * @@ -455,13 +467,28 @@ void svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber); * @return number of out of order segments */ u32 svm_fifo_n_ooo_segments (svm_fifo_t * f); -/* +/** * First out-of-order segment for fifo * * @param f fifo * @return first out-of-order segment for fifo */ ooo_segment_t *svm_fifo_first_ooo_segment (svm_fifo_t * f); +/** + * Check if fifo is sane. Debug only. + * + * @param f fifo + * @return 1 if sane, 0 otherwise + */ +u8 svm_fifo_is_sane (svm_fifo_t * f); +/** + * Declare this fifo is used by only a single thread. + * In this special case, fifo-growth can be done in an efficient way without delay. + * + * @param f fifo + * @return 1 if the fifo is already owned by another thread, 0 otherwise + */ +u8 svm_fifo_set_single_thread_owned (svm_fifo_t * f); format_function_t format_svm_fifo; /** @@ -635,7 +662,7 @@ svm_fifo_max_write_chunk (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_prod (f, &head, &tail); - return tail > head ? f->size - tail : f_free_count (f, head, tail); + return tail >= head ? f->size - tail : f_free_count (f, head, tail); } static inline u8 * @@ -740,95 +767,98 @@ svm_fifo_unset_event (svm_fifo_t * f) } /** - * Set specific want tx notification flag + * Set specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_deq_ntf_t * * @param f fifo * @param ntf_type type of notification requested */ static inline void -svm_fifo_add_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf |= ntf_type; + f->want_deq_ntf |= ntf_type; } /** - * Clear specific want tx notification flag + * Clear specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_ntf_t * * @param f fifo * @param ntf_type type of notification to be cleared */ static inline void -svm_fifo_del_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf &= ~ntf_type; + f->want_deq_ntf &= ~ntf_type; } /** - * Clear the want tx notification flag and set has tx notification + * Clear the want notification flag and set has notification * - * Should be used after enqueuing a tx event. This clears the - * SVM_FIFO_WANT_TX_NOTIF flag but it does not clear - * SVM_FIFO_WANT_TX_NOTIF_IF_FULL. If the latter was set, has_tx_ntf is - * set to avoid enqueueing tx events for for all dequeue operations until + * Should be used after enqueuing an event. This clears the + * SVM_FIFO_WANT_NOTIF flag but it does not clear + * SVM_FIFO_WANT_NOTIF_IF_FULL. If the latter was set, has_ntf is + * set to avoid enqueueing events for for all dequeue operations until * it is manually cleared. * * @param f fifo */ static inline void -svm_fifo_clear_tx_ntf (svm_fifo_t * f) +svm_fifo_clear_deq_ntf (svm_fifo_t * f) { - /* Set the flag if want_tx_notif_if_full was the only ntf requested */ - f->has_tx_ntf = f->want_tx_ntf == SVM_FIFO_WANT_TX_NOTIF_IF_FULL; - svm_fifo_del_want_tx_ntf (f, SVM_FIFO_WANT_TX_NOTIF); + /* Set the flag if want_notif_if_full was the only ntf requested */ + f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; + svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); } /** - * Clear has tx notification flag + * Clear has notification flag * - * The fifo generates only one event per SVM_FIFO_WANT_TX_NOTIF_IF_FULL - * request and sets has_tx_ntf. To received new events the flag must be + * The fifo generates only one event per SVM_FIFO_WANT_NOTIF_IF_FULL + * request and sets has_ntf. To received new events the flag must be * cleared using this function. * * @param f fifo */ static inline void -svm_fifo_reset_tx_ntf (svm_fifo_t * f) +svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) { - f->has_tx_ntf = 0; + f->has_deq_ntf = 0; } /** - * Check if fifo needs tx notification + * Check if fifo needs dequeue notification * - * Determines based on tx notification request flags and state of the fifo if - * a tx io event should be generated. + * Determines based on notification request flags and state of the fifo if + * an event should be generated. * * @param f fifo * @param n_last_deq number of bytes last dequeued - * @return 1 if tx io event should be generated, 0 otherwise + * @return 1 if event should be generated, 0 otherwise */ static inline u8 -svm_fifo_needs_tx_ntf (svm_fifo_t * f, u32 n_last_deq) +svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) { - u8 want_ntf = f->want_tx_ntf; + u8 want_ntf = f->want_deq_ntf; - if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_TX_NOTIF)) + if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)) return 0; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF) + else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF) return 1; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF_IF_FULL) + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) { u32 max_deq = svm_fifo_max_dequeue_cons (f); u32 nitems = f->nitems; - if (!f->has_tx_ntf && max_deq < nitems + if (!f->has_deq_ntf && max_deq < nitems && max_deq + n_last_deq >= nitems) return 1; - - return 0; + } + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) + { + if (!f->has_deq_ntf && svm_fifo_is_empty (f)) + return 1; } return 0; }