X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fsvm%2Fsvm_fifo.c;h=49b3d1728f31420d228c511d1748df5ded6a0b3c;hb=99e3b8b84e665616d5daa21d3f08c9c8358b8da4;hp=f79f37a4687958899b5d24ad11143a14ccc11f0c;hpb=af588825ba4586fadbdb742953ca640e94cd59ef;p=vpp.git diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index f79f37a4687..49b3d1728f3 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -21,11 +21,11 @@ #include #include -#define F_INVALID_CPTR (svm_fifo_chunk_ptr_t) ~0ULL +#define F_INVALID_CPTR (fs_sptr_t) ~0ULL CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len, - svm_fifo_chunk_ptr_t *last) + fs_sptr_t *last) { u32 n_chunk; @@ -56,7 +56,7 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f, CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len, - svm_fifo_chunk_ptr_t *last) + fs_sptr_t *last) { u32 n_chunk; @@ -72,8 +72,8 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, c = f_cptr (f, c->next); while ((to_copy -= n_chunk)) { - CLIB_MEM_UNPOISON (c, sizeof (*c)); - CLIB_MEM_UNPOISON (c->data, c->length); + clib_mem_unpoison (c, sizeof (*c)); + clib_mem_unpoison (c->data, c->length); n_chunk = clib_min (c->length, to_copy); clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk); c = c->length <= to_copy ? f_cptr (f, c->next) : c; @@ -91,7 +91,7 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, static inline void svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, - const u8 *src, u32 len, svm_fifo_chunk_ptr_t *last) + const u8 *src, u32 len, fs_sptr_t *last) { CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len, last); @@ -99,7 +99,7 @@ svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, static inline void svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, - u8 *dst, u32 len, svm_fifo_chunk_ptr_t *last) + u8 *dst, u32 len, fs_sptr_t *last) { CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len, last); @@ -398,6 +398,7 @@ svm_fifo_init (svm_fifo_t * f, u32 size) { c->start_byte = prev->start_byte + prev->length; c->enq_rb_index = c->deq_rb_index = RBTREE_TNIL_INDEX; + ASSERT (c->length >= 1 << FS_MIN_LOG2_CHUNK_SZ); prev = c; c = f_cptr (f, c->next); } @@ -905,7 +906,7 @@ int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src) { u32 tail, head, free_count, enq_pos; - svm_fifo_chunk_ptr_t last = F_INVALID_CPTR; + fs_sptr_t last = F_INVALID_CPTR; f_load_head_tail_prod (f, &head, &tail); @@ -1009,25 +1010,26 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], } else { - len = clib_min (free_count, len); + u32 n_left = clib_min (free_count, len); - if (f_pos_gt (tail + len, f_chunk_end (f_end_cptr (f)))) + if (f_pos_gt (tail + n_left, f_chunk_end (f_end_cptr (f)))) { - if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len))) + if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, n_left))) { - len = f_chunk_end (f_end_cptr (f)) - tail; - if (!len) + n_left = f_chunk_end (f_end_cptr (f)) - tail; + if (!n_left) return SVM_FIFO_EGROW; } } + len = n_left; i = 0; - while (len) + while (n_left) { - u32 to_copy = clib_min (segs[i].len, len); + u32 to_copy = clib_min (segs[i].len, n_left); svm_fifo_copy_to_chunk (f, f_tail_cptr (f), tail, segs[i].data, to_copy, &f->shr->tail_chunk); - len -= to_copy; + n_left -= to_copy; tail += to_copy; i++; } @@ -1140,7 +1142,7 @@ int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst) { u32 tail, head, cursize, head_idx; - svm_fifo_chunk_ptr_t last = F_INVALID_CPTR; + fs_sptr_t last = F_INVALID_CPTR; f_load_head_tail_cons (f, &head, &tail); @@ -1153,7 +1155,7 @@ svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst) len = clib_min (cursize - offset, len); head_idx = head + offset; - CLIB_MEM_UNPOISON (f->ooo_deq, sizeof (*f->ooo_deq)); + clib_mem_unpoison (f->ooo_deq, sizeof (*f->ooo_deq)); if (!f->ooo_deq || !f_chunk_includes_pos (f->ooo_deq, head_idx)) f_update_ooo_deq (f, head_idx, head_idx + len); @@ -1256,6 +1258,9 @@ svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs, if (n_avail < len && f_try_chunk_alloc (f, head, tail, len)) return SVM_FIFO_EGROW; + if (!fs || !n_segs) + return 0; + c = f_tail_cptr (f); head_pos = (tail - c->start_byte); fs[0].data = c->data + head_pos; @@ -1276,8 +1281,8 @@ svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs, } int -svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs, - u32 n_segs, u32 max_bytes) +svm_fifo_segments (svm_fifo_t *f, u32 offset, svm_fifo_seg_t *fs, u32 *n_segs, + u32 max_bytes) { u32 cursize, to_read, head, tail, fs_index = 1; u32 n_bytes, head_pos, len, start; @@ -1310,7 +1315,7 @@ svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs, fs[0].len = clib_min (c->length - head_pos, to_read); n_bytes = fs[0].len; - while (n_bytes < to_read && fs_index < n_segs) + while (n_bytes < to_read && fs_index < *n_segs) { c = f_cptr (f, c->next); len = clib_min (c->length, to_read - n_bytes); @@ -1319,6 +1324,7 @@ svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs, n_bytes += len; fs_index += 1; } + *n_segs = fs_index; return n_bytes; } @@ -1339,7 +1345,7 @@ svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf) ASSERT (svm_fifo_n_chunks (sf) == 1); clib_memcpy_fast (f_head_cptr (df)->data, f_head_cptr (sf)->data, - sf->shr->size); + f_head_cptr (sf)->length); f_load_head_tail_all_acq (sf, &head, &tail); clib_atomic_store_rel_n (&df->shr->head, head);