X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.h;h=ce4c53d9abebed2cde69cb057daa5958351b6b53;hb=1afe95272672ae0ff42109a9a682907fd169973e;hp=390e1170e27c18c70cb790ea7fb862249a226473;hpb=29a59c3ae18573043d9f9baa2796ab0b841bf6aa;p=vpp.git diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 390e1170e27..ce4c53d9abe 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -40,12 +40,13 @@ typedef struct #define SVM_FIFO_INVALID_INDEX ((u32)~0) #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 -typedef enum svm_fifo_tx_ntf_ +typedef enum svm_fifo_deq_ntf_ { - SVM_FIFO_NO_TX_NOTIF = 0, - SVM_FIFO_WANT_TX_NOTIF = 1, - SVM_FIFO_WANT_TX_NOTIF_IF_FULL = 2, -} svm_fifo_tx_ntf_t; + SVM_FIFO_NO_DEQ_NOTIF = 0, /**< No notification requested */ + SVM_FIFO_WANT_DEQ_NOTIF = 1, /**< Notify on dequeue */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL = 2, /**< Notify on transition from full */ + SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY = 4, /**< Notify on transition to empty */ +} svm_fifo_deq_ntf_t; typedef struct { @@ -64,9 +65,12 @@ typedef struct svm_fifo_chunk_ typedef enum svm_fifo_flag_ { - SVM_FIFO_F_SIZE_UPDATE = 1 << 0, - SVM_FIFO_F_MULTI_CHUNK = 1 << 1, - SVM_FIFO_F_LL_TRACKED = 1 << 2, + SVM_FIFO_F_MULTI_CHUNK = 1 << 0, + SVM_FIFO_F_GROW = 1 << 1, + SVM_FIFO_F_SHRINK = 1 << 2, + SVM_FIFO_F_COLLECT_CHUNKS = 1 << 3, + SVM_FIFO_F_LL_TRACKED = 1 << 4, + SVM_FIFO_F_SINGLE_THREAD_OWNED = 1 << 5, } svm_fifo_flag_t; typedef struct _svm_fifo @@ -86,19 +90,19 @@ typedef struct _svm_fifo u32 client_session_index; /**< app session index */ u8 master_thread_index; /**< session layer thread index */ u8 client_thread_index; /**< app worker index */ + i8 refcnt; /**< reference count */ u32 segment_manager; /**< session layer segment manager index */ u32 segment_index; /**< segment index in segment manager */ - u32 freelist_index; /**< aka log2(allocated_size) - const. */ - i8 refcnt; /**< reference count */ struct _svm_fifo *next; /**< next in freelist/active chain */ struct _svm_fifo *prev; /**< prev in active chain */ + u32 size_decrement; /**< bytes to remove from fifo */ CLIB_CACHE_LINE_ALIGN_MARK (consumer); u32 head; /**< fifo head position/byte */ svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */ svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ - volatile u32 want_tx_ntf; /**< producer wants nudge */ - volatile u32 has_tx_ntf; + volatile u32 want_deq_ntf; /**< producer wants nudge */ + volatile u32 has_deq_ntf; CLIB_CACHE_LINE_ALIGN_MARK (producer); u32 tail; /**< fifo tail position/byte */ @@ -114,7 +118,6 @@ typedef struct _svm_fifo svm_fifo_trace_elem_t *trace; #endif - svm_fifo_chunk_t default_chunk; } svm_fifo_t; typedef enum @@ -172,7 +175,8 @@ f_load_head_tail_prod (svm_fifo_t * f, u32 * head, u32 * tail) *head = clib_atomic_load_acq_n (&f->head); } -/* Load head and tail independent of producer/consumer role +/** + * Load head and tail independent of producer/consumer role * * Internal function. */ @@ -229,6 +233,13 @@ f_free_count (svm_fifo_t * f, u32 head, u32 tail) return (f->nitems - f_cursize (f, head, tail)); } +/** + * Try to shrink fifo size. + * + * Internal function. + */ +void svm_fifo_try_shrink (svm_fifo_t * f, u32 head, u32 tail); + /** * Create fifo of requested size * @@ -242,9 +253,16 @@ svm_fifo_t *svm_fifo_create (u32 size); /** * Initialize fifo * + * @param f fifo * @param size size for fifo */ void svm_fifo_init (svm_fifo_t * f, u32 size); +/** + * Initialize fifo chunks and rbtree + * + * @param f fifo + */ +void svm_fifo_init_chunks (svm_fifo_t * f); /** * Allocate a fifo chunk on heap * @@ -266,6 +284,31 @@ svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size); * @param c chunk or linked list of chunks to be added */ void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c); +/** + * Request to reduce fifo size by amount of bytes + * + * Because the producer might be enqueuing data when this is called, the + * actual size update is only applied when producer tries to enqueue new + * data, unless @param try_shrink is set. + * + * @param f fifo + * @param len number of bytes to remove from fifo. The actual number + * of bytes to be removed will be less or equal to this + * value. + * @param try_shrink flg to indicate if it's safe to try to shrink fifo + * size. It should be set only if this is called by the + * producer of if the producer is not using the fifo + * @return actual length fifo size will be reduced by + */ +int svm_fifo_reduce_size (svm_fifo_t * f, u32 len, u8 try_shrink); +/** + * Removes chunks that are after fifo end byte + * + * Needs to be called with segment heap pushed. + * + * @param f fifo + */ +svm_fifo_chunk_t *svm_fifo_collect_chunks (svm_fifo_t * f); /** * Free fifo and associated state * @@ -333,6 +376,16 @@ int svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src); */ int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src); + +/** + * Advance tail pointer + * + * Useful for moving tail pointer after external enqueue. + * + * @param f fifo + * @param len number of bytes to add to tail + */ +void svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len); /** * Overwrite fifo head with new data * @@ -411,13 +464,28 @@ void svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber); * @return number of out of order segments */ u32 svm_fifo_n_ooo_segments (svm_fifo_t * f); -/* +/** * First out-of-order segment for fifo * * @param f fifo * @return first out-of-order segment for fifo */ ooo_segment_t *svm_fifo_first_ooo_segment (svm_fifo_t * f); +/** + * Check if fifo is sane. Debug only. + * + * @param f fifo + * @return 1 if sane, 0 otherwise + */ +u8 svm_fifo_is_sane (svm_fifo_t * f); +/** + * Declare this fifo is used by only a single thread. + * In this special case, fifo-growth can be done in an efficient way without delay. + * + * @param f fifo + * @return 1 if the fifo is already owned by another thread, 0 otherwise + */ +u8 svm_fifo_set_single_thread_owned (svm_fifo_t * f); format_function_t format_svm_fifo; /** @@ -551,6 +619,8 @@ svm_fifo_max_enqueue_prod (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_prod (f, &head, &tail); + if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK)) + svm_fifo_try_shrink (f, head, tail); return f_free_count (f, head, tail); } @@ -565,6 +635,8 @@ svm_fifo_max_enqueue (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_all_acq (f, &head, &tail); + if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK)) + svm_fifo_try_shrink (f, head, tail); return f_free_count (f, head, tail); } @@ -587,26 +659,7 @@ svm_fifo_max_write_chunk (svm_fifo_t * f) { u32 head, tail; f_load_head_tail_prod (f, &head, &tail); - return tail > head ? f->size - tail : f_free_count (f, head, tail); -} - -/** - * Advance tail pointer - * - * Useful for moving tail pointer after external enqueue. - * - * @param f fifo - * @param len number of bytes to add to tail - */ -static inline void -svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len) -{ - ASSERT (len <= svm_fifo_max_enqueue_prod (f)); - /* load-relaxed: producer owned index */ - u32 tail = f->tail; - tail = (tail + len) % f->size; - /* store-rel: producer owned index (paired with load-acq in consumer) */ - clib_atomic_store_rel_n (&f->tail, tail); + return tail >= head ? f->size - tail : f_free_count (f, head, tail); } static inline u8 * @@ -711,95 +764,98 @@ svm_fifo_unset_event (svm_fifo_t * f) } /** - * Set specific want tx notification flag + * Set specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_deq_ntf_t * * @param f fifo * @param ntf_type type of notification requested */ static inline void -svm_fifo_add_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf |= ntf_type; + f->want_deq_ntf |= ntf_type; } /** - * Clear specific want tx notification flag + * Clear specific want notification flag * - * For list of flags see @ref svm_fifo_tx_ntf_t + * For list of flags see @ref svm_fifo_ntf_t * * @param f fifo * @param ntf_type type of notification to be cleared */ static inline void -svm_fifo_del_want_tx_ntf (svm_fifo_t * f, u8 ntf_type) +svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_tx_ntf &= ~ntf_type; + f->want_deq_ntf &= ~ntf_type; } /** - * Clear the want tx notification flag and set has tx notification + * Clear the want notification flag and set has notification * - * Should be used after enqueuing a tx event. This clears the - * SVM_FIFO_WANT_TX_NOTIF flag but it does not clear - * SVM_FIFO_WANT_TX_NOTIF_IF_FULL. If the latter was set, has_tx_ntf is - * set to avoid enqueueing tx events for for all dequeue operations until + * Should be used after enqueuing an event. This clears the + * SVM_FIFO_WANT_NOTIF flag but it does not clear + * SVM_FIFO_WANT_NOTIF_IF_FULL. If the latter was set, has_ntf is + * set to avoid enqueueing events for for all dequeue operations until * it is manually cleared. * * @param f fifo */ static inline void -svm_fifo_clear_tx_ntf (svm_fifo_t * f) +svm_fifo_clear_deq_ntf (svm_fifo_t * f) { - /* Set the flag if want_tx_notif_if_full was the only ntf requested */ - f->has_tx_ntf = f->want_tx_ntf == SVM_FIFO_WANT_TX_NOTIF_IF_FULL; - svm_fifo_del_want_tx_ntf (f, SVM_FIFO_WANT_TX_NOTIF); + /* Set the flag if want_notif_if_full was the only ntf requested */ + f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; + svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); } /** - * Clear has tx notification flag + * Clear has notification flag * - * The fifo generates only one event per SVM_FIFO_WANT_TX_NOTIF_IF_FULL - * request and sets has_tx_ntf. To received new events the flag must be + * The fifo generates only one event per SVM_FIFO_WANT_NOTIF_IF_FULL + * request and sets has_ntf. To received new events the flag must be * cleared using this function. * * @param f fifo */ static inline void -svm_fifo_reset_tx_ntf (svm_fifo_t * f) +svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) { - f->has_tx_ntf = 0; + f->has_deq_ntf = 0; } /** - * Check if fifo needs tx notification + * Check if fifo needs dequeue notification * - * Determines based on tx notification request flags and state of the fifo if - * a tx io event should be generated. + * Determines based on notification request flags and state of the fifo if + * an event should be generated. * * @param f fifo * @param n_last_deq number of bytes last dequeued - * @return 1 if tx io event should be generated, 0 otherwise + * @return 1 if event should be generated, 0 otherwise */ static inline u8 -svm_fifo_needs_tx_ntf (svm_fifo_t * f, u32 n_last_deq) +svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) { - u8 want_ntf = f->want_tx_ntf; + u8 want_ntf = f->want_deq_ntf; - if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_TX_NOTIF)) + if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)) return 0; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF) + else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF) return 1; - else if (want_ntf & SVM_FIFO_WANT_TX_NOTIF_IF_FULL) + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) { u32 max_deq = svm_fifo_max_dequeue_cons (f); u32 nitems = f->nitems; - if (!f->has_tx_ntf && max_deq < nitems + if (!f->has_deq_ntf && max_deq < nitems && max_deq + n_last_deq >= nitems) return 1; - - return 0; + } + if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) + { + if (!f->has_deq_ntf && svm_fifo_is_empty (f)) + return 1; } return 0; }