typedef enum svm_fifo_flag_
{
- SVM_FIFO_F_SIZE_UPDATE = 1 << 0,
- SVM_FIFO_F_MULTI_CHUNK = 1 << 1,
- SVM_FIFO_F_LL_TRACKED = 1 << 2,
+ SVM_FIFO_F_MULTI_CHUNK = 1 << 0,
+ SVM_FIFO_F_GROW = 1 << 1,
+ SVM_FIFO_F_SHRINK = 1 << 2,
+ SVM_FIFO_F_COLLECT_CHUNKS = 1 << 3,
+ SVM_FIFO_F_LL_TRACKED = 1 << 4,
} svm_fifo_flag_t;
typedef struct _svm_fifo
u32 client_session_index; /**< app session index */
u8 master_thread_index; /**< session layer thread index */
u8 client_thread_index; /**< app worker index */
+ i8 refcnt; /**< reference count */
u32 segment_manager; /**< session layer segment manager index */
u32 segment_index; /**< segment index in segment manager */
- u32 freelist_index; /**< aka log2(allocated_size) - const. */
- i8 refcnt; /**< reference count */
struct _svm_fifo *next; /**< next in freelist/active chain */
struct _svm_fifo *prev; /**< prev in active chain */
+ u32 size_decrement; /**< bytes to remove from fifo */
CLIB_CACHE_LINE_ALIGN_MARK (consumer);
u32 head; /**< fifo head position/byte */
svm_fifo_trace_elem_t *trace;
#endif
- svm_fifo_chunk_t default_chunk;
} svm_fifo_t;
typedef enum
*head = clib_atomic_load_acq_n (&f->head);
}
-/* Load head and tail independent of producer/consumer role
+/**
+ * Load head and tail independent of producer/consumer role
*
* Internal function.
*/
}
/**
- * Fifo free bytes, i.e., number of free bytes
+ * Distance to a from b, i.e., a - b in the fifo
*
- * Internal function
+ * Internal function.
*/
static inline u32
-f_free_count (svm_fifo_t * f, u32 head, u32 tail)
+f_distance_to (svm_fifo_t * f, u32 a, u32 b)
{
- return (f->nitems + head - tail);
+ return ((f->size + a - b) % f->size);
}
/**
- * Fifo current size, i.e., number of bytes enqueued
+ * Distance from a to b, i.e., b - a in the fifo
*
* Internal function.
*/
static inline u32
-f_cursize (svm_fifo_t * f, u32 head, u32 tail)
+f_distance_from (svm_fifo_t * f, u32 a, u32 b)
{
- return (f->nitems - f_free_count (f, head, tail));
+ return ((f->size + b - a) % f->size);
}
/**
- * Distance to a from b, i.e., a - b in the fifo
+ * Fifo current size, i.e., number of bytes enqueued
*
* Internal function.
*/
static inline u32
-f_distance_to (svm_fifo_t * f, u32 a, u32 b)
+f_cursize (svm_fifo_t * f, u32 head, u32 tail)
{
- return ((f->size + a - b) % f->size);
+ return (head <= tail ? tail - head : f->size + tail - head);
}
/**
- * Distance from a to b, i.e., b - a in the fifo
+ * Fifo free bytes, i.e., number of free bytes
*
- * Internal function.
+ * Internal function
*/
static inline u32
-f_distance_from (svm_fifo_t * f, u32 a, u32 b)
+f_free_count (svm_fifo_t * f, u32 head, u32 tail)
{
- return ((f->size + b - a) % f->size);
+ return (f->nitems - f_cursize (f, head, tail));
}
+/**
+ * Try to shrink fifo size.
+ *
+ * Internal function.
+ */
+void svm_fifo_try_shrink (svm_fifo_t * f, u32 head, u32 tail);
+
/**
* Create fifo of requested size
*
* @param c chunk or linked list of chunks to be added
*/
void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c);
+/**
+ * Request to reduce fifo size by amount of bytes
+ *
+ * Because the producer might be enqueuing data when this is called, the
+ * actual size update is only applied when producer tries to enqueue new
+ * data, unless @param try_shrink is set.
+ *
+ * @param f fifo
+ * @param len number of bytes to remove from fifo. The actual number
+ * of bytes to be removed will be less or equal to this
+ * value.
+ * @param try_shrink flg to indicate if it's safe to try to shrink fifo
+ * size. It should be set only if this is called by the
+ * producer of if the producer is not using the fifo
+ * @return actual length fifo size will be reduced by
+ */
+int svm_fifo_reduce_size (svm_fifo_t * f, u32 len, u8 try_shrink);
+/**
+ * Removes chunks that are after fifo end byte
+ *
+ * Needs to be called with segment heap pushed.
+ *
+ * @param f fifo
+ */
+svm_fifo_chunk_t *svm_fifo_collect_chunks (svm_fifo_t * f);
/**
* Free fifo and associated state
*
*/
int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len,
u8 * src);
+
+/**
+ * Advance tail pointer
+ *
+ * Useful for moving tail pointer after external enqueue.
+ *
+ * @param f fifo
+ * @param len number of bytes to add to tail
+ */
+void svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len);
/**
* Overwrite fifo head with new data
*
{
u32 head, tail;
f_load_head_tail_all_acq (f, &head, &tail);
- return head % f->size > tail % f->size;
+ return head > tail;
}
/**
{
u32 head, tail;
f_load_head_tail_prod (f, &head, &tail);
+ if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK))
+ svm_fifo_try_shrink (f, head, tail);
return f_free_count (f, head, tail);
}
{
u32 head, tail;
f_load_head_tail_all_acq (f, &head, &tail);
+ if (PREDICT_FALSE (f->flags & SVM_FIFO_F_SHRINK))
+ svm_fifo_try_shrink (f, head, tail);
return f_free_count (f, head, tail);
}
svm_fifo_max_read_chunk (svm_fifo_t * f)
{
u32 head, tail;
- u32 head_idx, tail_idx;
f_load_head_tail_cons (f, &head, &tail);
- head_idx = head % f->size;
- tail_idx = tail % f->size;
- return tail_idx > head_idx ? (tail_idx - head_idx) : (f->size - head_idx);
+ return tail >= head ? (tail - head) : (f->size - head);
}
/**
svm_fifo_max_write_chunk (svm_fifo_t * f)
{
u32 head, tail;
- u32 head_idx, tail_idx;
f_load_head_tail_prod (f, &head, &tail);
- head_idx = head % f->size;
- tail_idx = tail % f->size;
- return tail_idx >= head_idx ? (f->size - tail_idx) : (head_idx - tail_idx);
-}
-
-/**
- * Advance tail pointer
- *
- * Useful for moving tail pointer after external enqueue.
- *
- * @param f fifo
- * @param len number of bytes to add to tail
- */
-static inline void
-svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len)
-{
- ASSERT (len <= svm_fifo_max_enqueue_prod (f));
- /* load-relaxed: producer owned index */
- u32 tail = f->tail;
- tail += len;
- /* store-rel: producer owned index (paired with load-acq in consumer) */
- clib_atomic_store_rel_n (&f->tail, tail);
+ return tail > head ? f->size - tail : f_free_count (f, head, tail);
}
static inline u8 *
svm_fifo_head (svm_fifo_t * f)
{
/* load-relaxed: consumer owned index */
- return (f->head_chunk->data
- + ((f->head % f->size) - f->head_chunk->start_byte));
+ return (f->head_chunk->data + (f->head - f->head_chunk->start_byte));
}
static inline u8 *
svm_fifo_tail (svm_fifo_t * f)
{
/* load-relaxed: producer owned index */
- return (f->tail_chunk->data
- + ((f->tail % f->size) - f->tail_chunk->start_byte));
+ return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte));
}
static inline u8
Code Review / vpp.git / blobdiff