svm: change fifo pointers to offsets

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I113a630d6db4741f54e508f7e0ed77be139df0e4

diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c
index 862d81f..f54ed9f 100644 (file)
--- a/src/plugins/unittest/session_test.c
+++ b/src/plugins/unittest/session_test.c
@@ -1850,7 +1850,7 @@ session_test_mq_speed (vlib_main_t * vm, unformat_input_t * input)
   s.rx_fifo = rx_fifo;
   s.tx_fifo = tx_fifo;
   s.session_state = SESSION_STATE_READY;
-  counter = (u64 *) rx_fifo->shr->head_chunk->data;
+  counter = (u64 *) f_head_cptr (rx_fifo)->data;
   start = vlib_time_now (vm);
 
   pid = fork ();

diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c
index 978dc10..b798cc8 100644 (file)
--- a/src/plugins/unittest/svm_fifo_test.c
+++ b/src/plugins/unittest/svm_fifo_test.c
@@ -1412,13 +1412,13 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (rv == 2, "should have 2 chunks has %u", rv);
   SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
 
-  c = f->shr->head_chunk;
+  c = f_head_cptr (f);
   SFIFO_TEST (c->start_byte == 0, "head start byte should be %u", 0);
   SFIFO_TEST (c->length == 4096, "head chunk length should be %u", 4096);
   SFIFO_TEST (f->shr->tail_chunk == 0, "no tail chunk");
   SFIFO_TEST (f->ooo_enq == 0, "should have no ooo enq chunk");
   SFIFO_TEST (f->ooo_deq == 0, "should have no ooo deq chunk");
-  c = f->shr->end_chunk;
+  c = f_end_cptr (f);
   SFIFO_TEST (c->start_byte == last_start_byte, "end chunk start byte should"
              " be %u", last_start_byte);
   SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096);
@@ -1466,7 +1466,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk");
   SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk");
 
-  c = f->shr->end_chunk;
+  c = f_end_cptr (f);
   SFIFO_TEST (c->start_byte == last_start_byte,
              "end chunk should start at %u", last_start_byte);
   SFIFO_TEST (c->length == 8192, "end chunk length should be %u", 8192);
@@ -1580,7 +1580,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk");
   SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk");
 
-  c = f->shr->end_chunk;
+  c = f_end_cptr (f);
   SFIFO_TEST (c->start_byte == last_start_byte,
              "end chunk should start at %u", last_start_byte);
   SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384);
@@ -1623,7 +1623,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
 
   last_start_byte += 16384;
-  c = f->shr->end_chunk;
+  c = f_end_cptr (f);
   SFIFO_TEST (c->start_byte == last_start_byte,
              "end chunk should start at %u", last_start_byte);
   SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096);
@@ -1673,7 +1673,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
 
   last_start_byte += 4096;
-  c = f->shr->end_chunk;
+  c = f_end_cptr (f);
   SFIFO_TEST (c->start_byte == last_start_byte,
              "end chunk should start at %u", last_start_byte);
   SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384);
@@ -1941,12 +1941,13 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input)
   svm_fifo_set_size (f, fifo_size);
   validate_test_and_buf_vecs (&test_data, &data_buf, fifo_size);
 
-  c = f->shr->start_chunk;
+  c = f_start_cptr (f);
   SFIFO_TEST (c->next == 0, "no next");
 
   svm_fifo_fill_chunk_list (f);
   SFIFO_TEST (c->next != 0, "new chunk should've been allocated");
-  SFIFO_TEST (c->next->length == 4 << 20, "new chunk should be 4MB");
+  SFIFO_TEST (f_cptr (f, c->next)->length == 4 << 20,
+             "new chunk should be 4MB");
 
   rv = svm_fifo_max_write_chunk (f);
   SFIFO_TEST (rv == 4096, "max write chunk %u", rv);
@@ -1958,7 +1959,7 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane");
 
   c = svm_fifo_tail_chunk (f);
-  SFIFO_TEST (c == f->shr->end_chunk, "tail is end chunk");
+  SFIFO_TEST (c == f_end_cptr (f), "tail is end chunk");
 
   /* Initialize head chunk */
   rv = svm_fifo_max_read_chunk (f);
@@ -1972,7 +1973,7 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input)
   SFIFO_TEST (rv == 4096, "dequeue should work");
 
   c = svm_fifo_head_chunk (f);
-  SFIFO_TEST (c == f->shr->end_chunk, "head chunk should be last");
+  SFIFO_TEST (c == f_end_cptr (f), "head chunk should be last");
 
   rv = svm_fifo_max_read_chunk (f);
   SFIFO_TEST (rv == 0, "max read chunk %u", rv);

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 19fd052..008cca4 100644 (file)
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -148,28 +148,30 @@ fss_chunk_fl_index_is_valid (fifo_segment_slice_t * fss, u32 fl_index)
 }
 
 static void
-fss_chunk_free_list_push (fifo_segment_slice_t * fss, u32 fl_index,
-                         svm_fifo_chunk_t * c)
+fss_chunk_free_list_push (fifo_segment_header_t *fsh,
+                         fifo_segment_slice_t *fss, u32 fl_index,
+                         svm_fifo_chunk_t *c)
 {
   fss_chunk_freelist_lock (fss);
   c->next = fss->free_chunks[fl_index];
-  fss->free_chunks[fl_index] = c;
+  fss->free_chunks[fl_index] = fs_chunk_sptr (fsh, c);
   fss_chunk_freelist_unlock (fss);
 }
 
 static void
-fss_chunk_free_list_push_list (fifo_segment_slice_t * fss, u32 fl_index,
-                              svm_fifo_chunk_t * head,
-                              svm_fifo_chunk_t * tail)
+fss_chunk_free_list_push_list (fifo_segment_header_t *fsh,
+                              fifo_segment_slice_t *fss, u32 fl_index,
+                              svm_fifo_chunk_t *head, svm_fifo_chunk_t *tail)
 {
   fss_chunk_freelist_lock (fss);
   tail->next = fss->free_chunks[fl_index];
-  fss->free_chunks[fl_index] = head;
+  fss->free_chunks[fl_index] = fs_chunk_sptr (fsh, head);
   fss_chunk_freelist_unlock (fss);
 }
 
 static svm_fifo_chunk_t *
-fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index)
+fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
+                        u32 fl_index)
 {
   svm_fifo_chunk_t *c;
 
@@ -183,7 +185,7 @@ fss_chunk_free_list_pop (fifo_segment_slice_t * fss, u32 fl_index)
       return 0;
     }
 
-  c = fss->free_chunks[fl_index];
+  c = fs_chunk_ptr (fsh, fss->free_chunks[fl_index]);
   fss->free_chunks[fl_index] = c->next;
 
   fss_chunk_freelist_unlock (fss);
@@ -435,10 +437,10 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh,
 
   while (req_bytes)
     {
-      c = fss_chunk_free_list_pop (fss, fl_index);
+      c = fss_chunk_free_list_pop (fsh, fss, fl_index);
       if (c)
        {
-         c->next = first;
+         c->next = fs_chunk_sptr (fsh, first);
          first = c;
          n_alloc += fl_size;
          req_bytes -= clib_min (fl_size, req_bytes);
@@ -453,8 +455,8 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh,
              while (c)
                {
                  fl_index = fs_freelist_for_size (c->length);
-                 next = c->next;
-                 fss_chunk_free_list_push (fss, fl_index, c);
+                 next = fs_chunk_ptr (fsh, c->next);
+                 fss_chunk_free_list_push (fsh, fss, fl_index, c);
                  c = next;
                }
              n_alloc = 0;
@@ -464,7 +466,7 @@ fs_try_alloc_multi_chunk (fifo_segment_header_t * fsh,
              fl_index = fs_freelist_for_size (data_bytes) + 1;
              if (!fss_chunk_fl_index_is_valid (fss, fl_index))
                return 0;
-             first = fss_chunk_free_list_pop (fss, fl_index);
+             first = fss_chunk_free_list_pop (fsh, fss, fl_index);
              if (first)
                {
                  first->next = 0;
@@ -541,13 +543,13 @@ fsh_try_alloc_chunk_batch (fifo_segment_header_t * fsh,
     {
       c->start_byte = 0;
       c->length = rounded_data_size;
-      c->next = head;
+      c->next = fs_chunk_sptr (fsh, head);
       head = c;
       cmem += sizeof (*c) + rounded_data_size;
       c = (svm_fifo_chunk_t *) cmem;
     }
 
-  fss_chunk_free_list_push_list (fss, fl_index, head, tail);
+  fss_chunk_free_list_push_list (fsh, fss, fl_index, head, tail);
   fss->num_chunks[fl_index] += batch_size;
   fss_fl_chunk_bytes_add (fss, total_chunk_bytes);
   fsh_cached_bytes_add (fsh, total_chunk_bytes);
@@ -593,7 +595,7 @@ fsh_try_alloc_chunk (fifo_segment_header_t * fsh,
   fl_index = fs_freelist_for_size (data_bytes);
 
 free_list:
-  c = fss_chunk_free_list_pop (fss, fl_index);
+  c = fss_chunk_free_list_pop (fsh, fss, fl_index);
   if (c)
     {
       c->next = 0;
@@ -678,10 +680,10 @@ fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
       return 0;
     }
 
-  sf->start_chunk = c;
+  sf->start_chunk = fs_chunk_sptr (fsh, c);
   while (c->next)
-    c = c->next;
-  sf->end_chunk = c;
+    c = fs_chunk_ptr (fsh, c->next);
+  sf->end_chunk = fs_chunk_sptr (fsh, c);
   sf->size = data_bytes;
   sf->slice_index = slice_index;
 
@@ -710,9 +712,9 @@ fsh_slice_collect_chunks (fifo_segment_header_t * fsh,
   while (c)
     {
       CLIB_MEM_UNPOISON (c, sizeof (*c));
-      next = c->next;
+      next = fs_chunk_ptr (fsh, c->next);
       fl_index = fs_freelist_for_size (c->length);
-      fss_chunk_free_list_push (fss, fl_index, c);
+      fss_chunk_free_list_push (fsh, fss, fl_index, c);
       n_collect += fs_freelist_index_to_size (fl_index);
       c = next;
     }
@@ -842,7 +844,7 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
   fss = fsh_slice_get (fsh, sf->slice_index);
 
   /* Free fifo chunks */
-  fsh_slice_collect_chunks (fsh, fss, sf->start_chunk);
+  fsh_slice_collect_chunks (fsh, fss, fs_chunk_ptr (fsh, f->shr->start_chunk));
 
   sf->start_chunk = sf->end_chunk = 0;
   sf->head_chunk = sf->tail_chunk = 0;
@@ -895,12 +897,12 @@ fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f)
   if (f->flags & SVM_FIFO_F_LL_TRACKED)
     fss_fifo_del_active_list (fss, f);
 
-  c = f->shr->start_chunk;
+  c = fs_chunk_ptr (fs->h, f->shr->start_chunk);
   while (c)
     {
       fl_index = fs_freelist_for_size (c->length);
       clib_atomic_fetch_sub_rel (&fss->num_chunks[fl_index], 1);
-      c = c->next;
+      c = fs_chunk_ptr (fs->h, c->next);
     }
 }
 
@@ -918,12 +920,12 @@ fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f,
   if (f->flags & SVM_FIFO_F_LL_TRACKED)
     fss_fifo_add_active_list (fss, f);
 
-  c = f->shr->start_chunk;
+  c = fs_chunk_ptr (fs->h, f->shr->start_chunk);
   while (c)
     {
       fl_index = fs_freelist_for_size (c->length);
       clib_atomic_fetch_add_rel (&fss->num_chunks[fl_index], 1);
-      c = c->next;
+      c = fs_chunk_ptr (fs->h, c->next);
     }
 }
 
@@ -1068,7 +1070,8 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs)
 }
 
 static u32
-fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
+fs_slice_num_free_chunks (fifo_segment_header_t *fsh,
+                         fifo_segment_slice_t *fss, u32 size)
 {
   u32 count = 0, rounded_size, fl_index;
   svm_fifo_chunk_t *c;
@@ -1079,13 +1082,13 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
     {
       for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
        {
-         c = fss->free_chunks[i];
+         c = fs_chunk_ptr (fsh, fss->free_chunks[i]);
          if (c == 0)
            continue;
 
          while (c)
            {
-             c = c->next;
+             c = fs_chunk_ptr (fsh, c->next);
              count++;
            }
        }
@@ -1098,13 +1101,13 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
   if (fl_index >= FS_CHUNK_VEC_LEN)
     return 0;
 
-  c = fss->free_chunks[fl_index];
+  c = fs_chunk_ptr (fsh, fss->free_chunks[fl_index]);
   if (c == 0)
     return 0;
 
   while (c)
     {
-      c = c->next;
+      c = fs_chunk_ptr (fsh, c->next);
       count++;
     }
   return count;
@@ -1121,7 +1124,7 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
   for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
     {
       fss = fsh_slice_get (fsh, slice_index);
-      count += fs_slice_num_free_chunks (fss, size);
+      count += fs_slice_num_free_chunks (fsh, fss, size);
     }
   return count;
 }
@@ -1328,13 +1331,13 @@ format_fifo_segment (u8 * s, va_list * args)
       fss = fsh_slice_get (fsh, slice_index);
       for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
        {
-         c = fss->free_chunks[i];
+         c = fs_chunk_ptr (fsh, fss->free_chunks[i]);
          if (c == 0 && fss->num_chunks[i] == 0)
            continue;
          count = 0;
          while (c)
            {
-             c = c->next;
+             c = fs_chunk_ptr (fsh, c->next);
              count++;
            }
 

diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h
index 434b2c3..a65b3a8 100644 (file)
--- a/src/svm/fifo_types.h
+++ b/src/svm/fifo_types.h
@@ -33,16 +33,18 @@ STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS   7
 
 typedef struct fifo_segment_header_ fifo_segment_header_t;
+typedef struct svm_fifo_chunk_ svm_fifo_chunk_t;
+typedef svm_fifo_chunk_t *svm_fifo_chunk_ptr_t;
 
-typedef struct svm_fifo_chunk_
+struct svm_fifo_chunk_
 {
   u32 start_byte;              /**< chunk start byte */
   u32 length;                  /**< length of chunk in bytes */
-  struct svm_fifo_chunk_ *next;        /**< pointer to next chunk in linked-lists */
+  svm_fifo_chunk_ptr_t next;   /**< pointer to next chunk in linked-lists */
   rb_node_index_t enq_rb_index;        /**< enq node index if chunk in rbtree */
   rb_node_index_t deq_rb_index;        /**< deq node index if chunk in rbtree */
   u8 data[0];                  /**< start of chunk data */
-} svm_fifo_chunk_t;
+};
 
 typedef struct
 {
@@ -62,8 +64,8 @@ typedef struct
 typedef struct svm_fifo_shr_
 {
   CLIB_CACHE_LINE_ALIGN_MARK (shared);
-  svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */
-  svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */
+  svm_fifo_chunk_ptr_t start_chunk; /**< first chunk in fifo chunk list */
+  svm_fifo_chunk_ptr_t end_chunk;   /**< end chunk in fifo chunk list */
   volatile u32 has_event;      /**< non-zero if deq event exists */
   u32 min_alloc;               /**< min chunk alloc if space available */
   u32 size;                    /**< size of the fifo in bytes */
@@ -73,14 +75,14 @@ typedef struct svm_fifo_shr_
   struct svm_fifo_shr_ *next;  /**< next in freelist/active chain */
 
   CLIB_CACHE_LINE_ALIGN_MARK (consumer);
-  svm_fifo_chunk_t *head_chunk;        /**< tracks chunk where head lands */
+  svm_fifo_chunk_ptr_t head_chunk; /**< tracks chunk where head lands */
   u32 head;                    /**< fifo head position/byte */
   volatile u32 want_deq_ntf;   /**< producer wants nudge */
   volatile u32 has_deq_ntf;
 
   CLIB_CACHE_LINE_ALIGN_MARK (producer);
   u32 tail;                    /**< fifo tail position/byte */
-  svm_fifo_chunk_t *tail_chunk;        /**< tracks chunk where tail lands */
+  svm_fifo_chunk_ptr_t tail_chunk; /**< tracks chunk where tail lands */
   volatile u8 n_subscribers;   /**< Number of subscribers for io events */
   u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
 } svm_fifo_shared_t;
@@ -115,7 +117,8 @@ typedef struct _svm_fifo
 
 typedef struct fifo_segment_slice_
 {
-  svm_fifo_chunk_t *free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
+  svm_fifo_chunk_ptr_t
+    free_chunks[FS_CHUNK_VEC_LEN];     /**< Free chunks by size */
   svm_fifo_t *fifos;                   /**< Linked list of active RX fifos */
   svm_fifo_shared_t *free_fifos;       /**< Freelists of fifo shared hdrs  */
   uword n_fl_chunk_bytes;              /**< Chunk bytes on freelist */
@@ -153,6 +156,18 @@ struct fifo_segment_header_
 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
                             int n_bytes);
 
+always_inline svm_fifo_chunk_t *
+fs_chunk_ptr (fifo_segment_header_t *fsh, svm_fifo_chunk_ptr_t cp)
+{
+  return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + pointer_to_uword (cp)) : 0;
+}
+
+always_inline svm_fifo_chunk_ptr_t
+fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
+{
+  return c ? (svm_fifo_chunk_ptr_t) ((u8 *) c - (u8 *) fsh) : 0;
+}
+
 #endif /* SRC_SVM_FIFO_TYPES_H_ */
 
 /*

diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 2f910e0..f79f37a 100644 (file)
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -21,9 +21,11 @@
 #include <svm/fifo_segment.h>
 #include <vppinfra/cpu.h>
 
-CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t * f,
-              svm_fifo_chunk_t * c, u32 tail_idx, const u8 * src, u32 len,
-              svm_fifo_chunk_t ** last)
+#define F_INVALID_CPTR (svm_fifo_chunk_ptr_t) ~0ULL
+
+CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f,
+              svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len,
+              svm_fifo_chunk_ptr_t *last)
 {
   u32 n_chunk;
 
@@ -36,15 +38,15 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t * f,
     {
       u32 to_copy = len;
       clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
-      c = c->next;
+      c = f_cptr (f, c->next);
       while ((to_copy -= n_chunk))
        {
          n_chunk = clib_min (c->length, to_copy);
          clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
-         c = c->length <= to_copy ? c->next : c;
+         c = c->length <= to_copy ? f_cptr (f, c->next) : c;
        }
       if (*last)
-       *last = c;
+       *last = f_csptr (f, c);
     }
   else
     {
@@ -52,9 +54,9 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t * f,
     }
 }
 
-CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t * f,
-              svm_fifo_chunk_t * c, u32 head_idx, u8 * dst, u32 len,
-              svm_fifo_chunk_t ** last)
+CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f,
+              svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len,
+              svm_fifo_chunk_ptr_t *last)
 {
   u32 n_chunk;
 
@@ -67,17 +69,17 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t * f,
     {
       u32 to_copy = len;
       clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
-      c = c->next;
+      c = f_cptr (f, c->next);
       while ((to_copy -= n_chunk))
        {
          CLIB_MEM_UNPOISON (c, sizeof (*c));
          CLIB_MEM_UNPOISON (c->data, c->length);
          n_chunk = clib_min (c->length, to_copy);
          clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
-         c = c->length <= to_copy ? c->next : c;
+         c = c->length <= to_copy ? f_cptr (f, c->next) : c;
        }
       if (*last)
-       *last = c;
+       *last = f_csptr (f, c);
     }
   else
     {
@@ -88,16 +90,16 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t * f,
 #ifndef CLIB_MARCH_VARIANT
 
 static inline void
-svm_fifo_copy_to_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 tail_idx,
-                       const u8 * src, u32 len, svm_fifo_chunk_t ** last)
+svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx,
+                       const u8 *src, u32 len, svm_fifo_chunk_ptr_t *last)
 {
   CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len,
                                                 last);
 }
 
 static inline void
-svm_fifo_copy_from_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c, u32 head_idx,
-                         u8 * dst, u32 len, svm_fifo_chunk_t ** last)
+svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx,
+                         u8 *dst, u32 len, svm_fifo_chunk_ptr_t *last)
 {
   CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len,
                                                   last);
@@ -387,17 +389,17 @@ svm_fifo_init (svm_fifo_t * f, u32 size)
   /*
    * Initialize chunks
    */
-  f->shr->start_chunk->start_byte = 0;
-  prev = f->shr->start_chunk;
+  prev = f_start_cptr (f);
+  prev->start_byte = 0;
   prev->enq_rb_index = prev->deq_rb_index = RBTREE_TNIL_INDEX;
-  c = prev->next;
+  c = f_cptr (f, prev->next);
 
   while (c)
     {
       c->start_byte = prev->start_byte + prev->length;
       c->enq_rb_index = c->deq_rb_index = RBTREE_TNIL_INDEX;
       prev = c;
-      c = c->next;
+      c = f_cptr (f, c->next);
     }
 }
 
@@ -447,7 +449,7 @@ svm_fifo_alloc (u32 data_size_in_bytes)
   c->length = data_size_in_bytes;
   c->enq_rb_index = RBTREE_TNIL_INDEX;
   c->deq_rb_index = RBTREE_TNIL_INDEX;
-  f->shr->start_chunk = f->shr->end_chunk = c;
+  f->shr->start_chunk = f->shr->end_chunk = f_csptr (f, c);
 
   return f;
 }
@@ -486,9 +488,9 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos)
 {
   svm_fifo_chunk_t *c;
 
-  c = f->shr->start_chunk;
+  c = f_start_cptr (f);
   while (c && !f_chunk_includes_pos (c, pos))
-    c = c->next;
+    c = f_cptr (f, c->next);
 
   return c;
 }
@@ -502,7 +504,7 @@ svm_fifo_find_next_chunk (svm_fifo_t * f, svm_fifo_chunk_t * start, u32 pos)
 
   c = start;
   while (c && !f_chunk_includes_pos (c, pos))
-    c = c->next;
+    c = f_cptr (f, c->next);
 
   return c;
 }
@@ -513,17 +515,16 @@ svm_fifo_max_read_chunk (svm_fifo_t * f)
   u32 head, tail, end_chunk;
 
   f_load_head_tail_cons (f, &head, &tail);
-  ASSERT (!f->shr->head_chunk ||
-         f_chunk_includes_pos (f->shr->head_chunk, head));
+  ASSERT (!f->shr->head_chunk || f_chunk_includes_pos (f_head_cptr (f), head));
 
   if (!f->shr->head_chunk)
     {
-      f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+      f->shr->head_chunk = f_csptr (f, svm_fifo_find_chunk (f, head));
       if (PREDICT_FALSE (!f->shr->head_chunk))
        return 0;
     }
 
-  end_chunk = f_chunk_end (f->shr->head_chunk);
+  end_chunk = f_chunk_end (f_head_cptr (f));
 
   return f_pos_lt (end_chunk, tail) ? end_chunk - head : tail - head;
 }
@@ -531,13 +532,15 @@ svm_fifo_max_read_chunk (svm_fifo_t * f)
 u32
 svm_fifo_max_write_chunk (svm_fifo_t * f)
 {
+  svm_fifo_chunk_t *tail_chunk;
   u32 head, tail;
 
   f_load_head_tail_prod (f, &head, &tail);
-  ASSERT (!f->shr->tail_chunk ||
-         f_chunk_includes_pos (f->shr->tail_chunk, tail));
+  tail_chunk = f_tail_cptr (f);
+
+  ASSERT (!tail_chunk || f_chunk_includes_pos (tail_chunk, tail));
 
-  return f->shr->tail_chunk ? f_chunk_end (f->shr->tail_chunk) - tail : 0;
+  return tail_chunk ? f_chunk_end (tail_chunk) - tail : 0;
 }
 
 static rb_node_t *
@@ -607,13 +610,13 @@ f_update_ooo_enq (svm_fifo_t * f, u32 start_pos, u32 end_pos)
   /* Use linear search if rbtree is not initialized */
   if (PREDICT_FALSE (!rb_tree_is_init (rt)))
     {
-      f->ooo_enq = svm_fifo_find_next_chunk (f, f->shr->tail_chunk, start_pos);
+      f->ooo_enq = svm_fifo_find_next_chunk (f, f_tail_cptr (f), start_pos);
       return;
     }
 
   if (rt->root == RBTREE_TNIL_INDEX)
     {
-      c = f->shr->tail_chunk;
+      c = f_tail_cptr (f);
       ASSERT (c->enq_rb_index == RBTREE_TNIL_INDEX);
       c->enq_rb_index = rb_tree_add_custom (rt, c->start_byte,
                                            pointer_to_uword (c), f_pos_lt);
@@ -633,7 +636,7 @@ f_update_ooo_enq (svm_fifo_t * f, u32 start_pos, u32 end_pos)
 
   do
     {
-      c = c->next;
+      c = f_cptr (f, c->next);
       if (!c || c->enq_rb_index != RBTREE_TNIL_INDEX)
        break;
 
@@ -662,7 +665,7 @@ f_update_ooo_deq (svm_fifo_t * f, u32 start_pos, u32 end_pos)
 
   if (rt->root == RBTREE_TNIL_INDEX)
     {
-      c = f->shr->start_chunk;
+      c = f_start_cptr (f);
       ASSERT (c->deq_rb_index == RBTREE_TNIL_INDEX);
       c->deq_rb_index = rb_tree_add_custom (rt, c->start_byte,
                                            pointer_to_uword (c), f_pos_lt);
@@ -682,7 +685,7 @@ f_update_ooo_deq (svm_fifo_t * f, u32 start_pos, u32 end_pos)
 
   do
     {
-      c = c->next;
+      c = f_cptr (f, c->next);
       if (!c || c->deq_rb_index != RBTREE_TNIL_INDEX)
        break;
 
@@ -713,7 +716,7 @@ f_lookup_clear_enq_chunks (svm_fifo_t * f, svm_fifo_chunk_t * start,
          c->enq_rb_index = RBTREE_TNIL_INDEX;
        }
 
-      c = c->next;
+      c = f_cptr (f, c->next);
     }
 
   /* No ooo segments left, so make sure the current chunk
@@ -747,7 +750,7 @@ f_lookup_clear_deq_chunks (svm_fifo_t * f, svm_fifo_chunk_t * start,
          c->deq_rb_index = RBTREE_TNIL_INDEX;
        }
 
-      c = c->next;
+      c = f_cptr (f, c->next);
     }
 
   return c;
@@ -785,18 +788,19 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * src, u32 len)
   f_load_head_tail_cons (f, &head, &tail);
 
   if (!f->shr->head_chunk)
-    f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+    f->shr->head_chunk = f_csptr (f, svm_fifo_find_chunk (f, head));
 
-  c = f->shr->head_chunk;
+  c = f_head_cptr (f);
   head_idx = head - c->start_byte;
   n_chunk = c->length - head_idx;
   if (len <= n_chunk)
     clib_memcpy_fast (&c->data[head_idx], src, len);
   else
     {
-      ASSERT (len - n_chunk <= c->next->length);
+      ASSERT (len - n_chunk <= f_cptr (f, c->next)->length);
       clib_memcpy_fast (&c->data[head_idx], src, n_chunk);
-      clib_memcpy_fast (&c->next->data[0], src + n_chunk, len - n_chunk);
+      clib_memcpy_fast (&f_cptr (f, c->next)->data[0], src + n_chunk,
+                       len - n_chunk);
     }
 }
 
@@ -806,7 +810,8 @@ f_try_chunk_alloc (svm_fifo_t * f, u32 head, u32 tail, u32 len)
   svm_fifo_chunk_t *c, *cur, *prev;
   u32 alloc_size, free_alloced;
 
-  free_alloced = f_chunk_end (f->shr->end_chunk) - tail;
+  prev = f_end_cptr (f);
+  free_alloced = f_chunk_end (prev) - tail;
 
   alloc_size = clib_min (f->shr->min_alloc, f->shr->size - (tail - head));
   alloc_size = clib_max (alloc_size, len - free_alloced);
@@ -816,7 +821,6 @@ f_try_chunk_alloc (svm_fifo_t * f, u32 head, u32 tail, u32 len)
     return -1;
 
   cur = c;
-  prev = f->shr->end_chunk;
 
   while (cur)
     {
@@ -825,15 +829,15 @@ f_try_chunk_alloc (svm_fifo_t * f, u32 head, u32 tail, u32 len)
       cur->deq_rb_index = RBTREE_TNIL_INDEX;
 
       prev = cur;
-      cur = cur->next;
+      cur = f_cptr (f, cur->next);
     }
 
+  f_csptr_link (f, f->shr->end_chunk, c);
   prev->next = 0;
-  f->shr->end_chunk->next = c;
-  f->shr->end_chunk = prev;
+  f->shr->end_chunk = f_csptr (f, prev);
 
   if (!f->shr->tail_chunk)
-    f->shr->tail_chunk = c;
+    f->shr->tail_chunk = f_csptr (f, c);
 
   return 0;
 }
@@ -857,20 +861,19 @@ svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src)
   /* number of bytes we're going to copy */
   len = clib_min (free_count, len);
 
-  if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
+  if (f_pos_gt (tail + len, f_chunk_end (f_end_cptr (f))))
     {
       if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
        {
-         len = f_chunk_end (f->shr->end_chunk) - tail;
+         len = f_chunk_end (f_end_cptr (f)) - tail;
          if (!len)
            return SVM_FIFO_EGROW;
        }
     }
 
-  old_tail_c = f->shr->tail_chunk;
+  old_tail_c = f_tail_cptr (f);
 
-  svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, src, len,
-                         &f->shr->tail_chunk);
+  svm_fifo_copy_to_chunk (f, old_tail_c, tail, src, len, &f->shr->tail_chunk);
   tail = tail + len;
 
   svm_fifo_trace_add (f, head, len, 2);
@@ -880,7 +883,8 @@ svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src)
     {
       len += ooo_segment_try_collect (f, len, &tail);
       /* Tail chunk might've changed even if nothing was collected */
-      f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
+      f->shr->tail_chunk =
+       f_csptr (f, f_lookup_clear_enq_chunks (f, old_tail_c, tail));
       f->ooo_enq = 0;
     }
 
@@ -901,6 +905,7 @@ int
 svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
 {
   u32 tail, head, free_count, enq_pos;
+  svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
 
   f_load_head_tail_prod (f, &head, &tail);
 
@@ -914,7 +919,7 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
 
   enq_pos = tail + offset;
 
-  if (f_pos_gt (enq_pos + len, f_chunk_end (f->shr->end_chunk)))
+  if (f_pos_gt (enq_pos + len, f_chunk_end (f_end_cptr (f))))
     {
       if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, offset + len)))
        return SVM_FIFO_EGROW;
@@ -926,7 +931,9 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
   if (!f->ooo_enq || !f_chunk_includes_pos (f->ooo_enq, enq_pos))
     f_update_ooo_enq (f, enq_pos, enq_pos + len);
 
-  svm_fifo_copy_to_chunk (f, f->ooo_enq, enq_pos, src, len, &f->ooo_enq);
+  svm_fifo_copy_to_chunk (f, f->ooo_enq, enq_pos, src, len, &last);
+  if (last != F_INVALID_CPTR)
+    f->ooo_enq = f_cptr (f, last);
 
   return 0;
 }
@@ -947,13 +954,13 @@ svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len)
   if (rb_tree_is_init (&f->ooo_enq_lookup))
     {
       f->shr->tail_chunk =
-       f_lookup_clear_enq_chunks (f, f->shr->tail_chunk, tail);
+       f_csptr (f, f_lookup_clear_enq_chunks (f, f_tail_cptr (f), tail));
       f->ooo_enq = 0;
     }
   else
     {
       f->shr->tail_chunk =
-       svm_fifo_find_next_chunk (f, f->shr->tail_chunk, tail);
+       f_csptr (f, svm_fifo_find_next_chunk (f, f_tail_cptr (f), tail));
     }
 
   /* store-rel: producer owned index (paired with load-acq in consumer) */
@@ -980,14 +987,14 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
   for (i = 0; i < n_segs; i++)
     len += segs[i].len;
 
-  old_tail_c = f->shr->tail_chunk;
+  old_tail_c = f_tail_cptr (f);
 
   if (!allow_partial)
     {
       if (PREDICT_FALSE (free_count < len))
        return SVM_FIFO_EFULL;
 
-      if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
+      if (f_pos_gt (tail + len, f_chunk_end (f_end_cptr (f))))
        {
          if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
            return SVM_FIFO_EGROW;
@@ -995,7 +1002,7 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
 
       for (i = 0; i < n_segs; i++)
        {
-         svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data,
+         svm_fifo_copy_to_chunk (f, f_tail_cptr (f), tail, segs[i].data,
                                  segs[i].len, &f->shr->tail_chunk);
          tail += segs[i].len;
        }
@@ -1004,11 +1011,11 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
     {
       len = clib_min (free_count, len);
 
-      if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk)))
+      if (f_pos_gt (tail + len, f_chunk_end (f_end_cptr (f))))
        {
          if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
            {
-             len = f_chunk_end (f->shr->end_chunk) - tail;
+             len = f_chunk_end (f_end_cptr (f)) - tail;
              if (!len)
                return SVM_FIFO_EGROW;
            }
@@ -1018,7 +1025,7 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
       while (len)
        {
          u32 to_copy = clib_min (segs[i].len, len);
-         svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data,
+         svm_fifo_copy_to_chunk (f, f_tail_cptr (f), tail, segs[i].data,
                                  to_copy, &f->shr->tail_chunk);
          len -= to_copy;
          tail += to_copy;
@@ -1031,7 +1038,8 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
     {
       len += ooo_segment_try_collect (f, len, &tail);
       /* Tail chunk might've changed even if nothing was collected */
-      f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
+      f->shr->tail_chunk =
+       f_csptr (f, f_lookup_clear_enq_chunks (f, old_tail_c, tail));
       f->ooo_enq = 0;
     }
 
@@ -1048,12 +1056,11 @@ f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo)
   rb_tree_t *rt;
   rb_node_t *n;
 
-  ASSERT (!f_chunk_includes_pos (f->shr->start_chunk, end_pos));
-
   if (maybe_ooo)
     rt = &f->ooo_deq_lookup;
 
-  c = f->shr->start_chunk;
+  c = f_start_cptr (f);
+  ASSERT (!f_chunk_includes_pos (c, end_pos));
 
   do
     {
@@ -1067,7 +1074,7 @@ f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo)
       if (!c->next)
        break;
       prev = c;
-      c = c->next;
+      c = f_cptr (f, c->next);
     }
   while (!f_chunk_includes_pos (c, end_pos));
 
@@ -1087,8 +1094,8 @@ f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo)
     return 0;
 
   prev->next = 0;
-  start = f->shr->start_chunk;
-  f->shr->start_chunk = c;
+  start = f_start_cptr (f);
+  f->shr->start_chunk = f_csptr (f, c);
 
   return start;
 }
@@ -1109,9 +1116,9 @@ svm_fifo_dequeue (svm_fifo_t * f, u32 len, u8 * dst)
   len = clib_min (cursize, len);
 
   if (!f->shr->head_chunk)
-    f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+    f->shr->head_chunk = f_csptr (f, svm_fifo_find_chunk (f, head));
 
-  svm_fifo_copy_from_chunk (f, f->shr->head_chunk, head, dst, len,
+  svm_fifo_copy_from_chunk (f, f_head_cptr (f), head, dst, len,
                            &f->shr->head_chunk);
   head = head + len;
 
@@ -1119,7 +1126,7 @@ svm_fifo_dequeue (svm_fifo_t * f, u32 len, u8 * dst)
    * Use svm_fifo_dequeue_drop instead. */
   ASSERT (rb_tree_n_nodes (&f->ooo_deq_lookup) <= 1);
 
-  if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk)))
+  if (f_pos_geq (head, f_chunk_end (f_start_cptr (f))))
     fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
                        f_unlink_chunks (f, head, 0));
 
@@ -1133,6 +1140,7 @@ int
 svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst)
 {
   u32 tail, head, cursize, head_idx;
+  svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
 
   f_load_head_tail_cons (f, &head, &tail);
 
@@ -1149,7 +1157,9 @@ svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst)
   if (!f->ooo_deq || !f_chunk_includes_pos (f->ooo_deq, head_idx))
     f_update_ooo_deq (f, head_idx, head_idx + len);
 
-  svm_fifo_copy_from_chunk (f, f->ooo_deq, head_idx, dst, len, &f->ooo_deq);
+  svm_fifo_copy_from_chunk (f, f->ooo_deq, head_idx, dst, len, &last);
+  if (last != F_INVALID_CPTR)
+    f->ooo_deq = f_cptr (f, last);
   return len;
 }
 
@@ -1173,11 +1183,11 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 len)
   /* move head */
   head = head + total_drop_bytes;
 
-  if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk)))
+  if (f_pos_geq (head, f_chunk_end (f_start_cptr (f))))
     {
       fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
                          f_unlink_chunks (f, head, 1));
-      f->shr->head_chunk = f_chunk_includes_pos (f->shr->start_chunk, head) ?
+      f->shr->head_chunk = f_chunk_includes_pos (f_start_cptr (f), head) ?
                             f->shr->start_chunk :
                             0;
     }
@@ -1199,12 +1209,13 @@ svm_fifo_dequeue_drop_all (svm_fifo_t * f)
 
   f_load_head_tail_all_acq (f, &head, &tail);
 
-  if (!f->shr->head_chunk || !f_chunk_includes_pos (f->shr->head_chunk, head))
-    f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+  if (!f->shr->head_chunk || !f_chunk_includes_pos (f_head_cptr (f), head))
+    f->shr->head_chunk = f_csptr (f, svm_fifo_find_chunk (f, head));
 
-  f->shr->head_chunk = f_lookup_clear_deq_chunks (f, f->shr->head_chunk, tail);
+  f->shr->head_chunk =
+    f_csptr (f, f_lookup_clear_deq_chunks (f, f_head_cptr (f), tail));
 
-  if (f_pos_geq (tail, f_chunk_end (f->shr->start_chunk)))
+  if (f_pos_geq (tail, f_chunk_end (f_start_cptr (f))))
     fsh_collect_chunks (f->fs_hdr, f->shr->slice_index,
                        f_unlink_chunks (f, tail, 0));
 
@@ -1219,7 +1230,7 @@ svm_fifo_fill_chunk_list (svm_fifo_t * f)
 
   f_load_head_tail_prod (f, &head, &tail);
 
-  if (f_chunk_end (f->shr->end_chunk) - head >= f->shr->size)
+  if (f_chunk_end (f_end_cptr (f)) - head >= f->shr->size)
     return 0;
 
   if (f_try_chunk_alloc (f, head, tail, f->shr->size - (tail - head)))
@@ -1240,12 +1251,12 @@ svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs,
   if (f_free_count (f, head, tail) < len)
     return SVM_FIFO_EFULL;
 
-  n_avail = f_chunk_end (f->shr->end_chunk) - tail;
+  n_avail = f_chunk_end (f_end_cptr (f)) - tail;
 
   if (n_avail < len && f_try_chunk_alloc (f, head, tail, len))
     return SVM_FIFO_EGROW;
 
-  c = f->shr->tail_chunk;
+  c = f_tail_cptr (f);
   head_pos = (tail - c->start_byte);
   fs[0].data = c->data + head_pos;
   fs[0].len = clib_min (c->length - head_pos, len);
@@ -1253,7 +1264,7 @@ svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs,
 
   while (n_bytes < len && fs_index < n_segs)
     {
-      c = c->next;
+      c = f_cptr (f, c->next);
       clen = clib_min (c->length, len - n_bytes);
       fs[fs_index].data = c->data;
       fs[fs_index].len = clen;
@@ -1287,12 +1298,12 @@ svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs,
   start = head + offset;
 
   if (!f->shr->head_chunk)
-    f->shr->head_chunk = svm_fifo_find_chunk (f, head);
+    f->shr->head_chunk = f_csptr (f, svm_fifo_find_chunk (f, head));
 
-  c = f->shr->head_chunk;
+  c = f_head_cptr (f);
 
   while (!f_chunk_includes_pos (c, start))
-    c = c->next;
+    c = f_cptr (f, c->next);
 
   head_pos = start - c->start_byte;
   fs[0].data = c->data + head_pos;
@@ -1301,7 +1312,7 @@ svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs,
 
   while (n_bytes < to_read && fs_index < n_segs)
     {
-      c = c->next;
+      c = f_cptr (f, c->next);
       len = clib_min (c->length, to_read - n_bytes);
       fs[fs_index].data = c->data;
       fs[fs_index].len = len;
@@ -1327,7 +1338,7 @@ svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf)
   /* Support only single chunk clones for now */
   ASSERT (svm_fifo_n_chunks (sf) == 1);
 
-  clib_memcpy_fast (df->shr->head_chunk->data, sf->shr->head_chunk->data,
+  clib_memcpy_fast (f_head_cptr (df)->data, f_head_cptr (sf)->data,
                    sf->shr->size);
 
   f_load_head_tail_all_acq (sf, &head, &tail);
@@ -1360,10 +1371,12 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail)
 
   c = svm_fifo_find_chunk (f, head);
   ASSERT (c != 0);
-  f->shr->head_chunk = f->ooo_deq = c;
+  f->ooo_deq = c;
+  f->shr->head_chunk = f_csptr (f, c);
   c = svm_fifo_find_chunk (f, tail);
   ASSERT (c != 0);
-  f->shr->tail_chunk = f->ooo_enq = c;
+  f->ooo_enq = c;
+  f->shr->tail_chunk = f_csptr (f, c);
 }
 
 void
@@ -1395,19 +1408,18 @@ svm_fifo_is_sane (svm_fifo_t * f)
   svm_fifo_chunk_t *tmp;
 
   if (f->shr->head_chunk &&
-      !f_chunk_includes_pos (f->shr->head_chunk, f->shr->head))
+      !f_chunk_includes_pos (f_head_cptr (f), f->shr->head))
     return 0;
   if (f->shr->tail_chunk &&
-      !f_chunk_includes_pos (f->shr->tail_chunk, f->shr->tail))
+      !f_chunk_includes_pos (f_tail_cptr (f), f->shr->tail))
     return 0;
   if (f->ooo_deq)
     {
       if (rb_tree_is_init (&f->ooo_deq_lookup))
        {
          if (f_pos_lt (f->ooo_deq->start_byte,
-                       f->shr->start_chunk->start_byte) ||
-             f_pos_gt (f->ooo_deq->start_byte,
-                       f_chunk_end (f->shr->end_chunk)))
+                       f_start_cptr (f)->start_byte) ||
+             f_pos_gt (f->ooo_deq->start_byte, f_chunk_end (f_end_cptr (f))))
            return 0;
 
          tmp = f_find_chunk_rbtree (&f->ooo_deq_lookup,
@@ -1423,9 +1435,8 @@ svm_fifo_is_sane (svm_fifo_t * f)
       if (rb_tree_is_init (&f->ooo_enq_lookup))
        {
          if (f_pos_lt (f->ooo_enq->start_byte,
-                       f->shr->start_chunk->start_byte) ||
-             f_pos_gt (f->ooo_enq->start_byte,
-                       f_chunk_end (f->shr->end_chunk)))
+                       f_start_cptr (f)->start_byte) ||
+             f_pos_gt (f->ooo_enq->start_byte, f_chunk_end (f_end_cptr (f))))
            return 0;
 
          tmp = f_find_chunk_rbtree (&f->ooo_enq_lookup,
@@ -1433,19 +1444,19 @@ svm_fifo_is_sane (svm_fifo_t * f)
        }
       else
        {
-         tmp = svm_fifo_find_next_chunk (f, f->shr->tail_chunk,
+         tmp = svm_fifo_find_next_chunk (f, f_tail_cptr (f),
                                          f->ooo_enq->start_byte);
        }
       if (tmp != f->ooo_enq)
        return 0;
     }
 
-  if (f->shr->start_chunk->next)
+  if (f_start_cptr (f)->next)
     {
       svm_fifo_chunk_t *c, *prev = 0, *tmp;
       u32 chunks_bytes = 0;
 
-      c = f->shr->start_chunk;
+      c = f_start_cptr (f);
       do
        {
          tmp = svm_fifo_find_chunk (f, c->start_byte);
@@ -1475,7 +1486,7 @@ svm_fifo_is_sane (svm_fifo_t * f)
 
          chunks_bytes += c->length;
          prev = c;
-         c = c->next;
+         c = f_cptr (f, c->next);
        }
       while (c);
 
@@ -1492,11 +1503,11 @@ svm_fifo_n_chunks (svm_fifo_t * f)
   svm_fifo_chunk_t *c;
   int n_chunks = 0;
 
-  c = f->shr->start_chunk;
+  c = f_start_cptr (f);
   while (c)
     {
       n_chunks++;
-      c = c->next;
+      c = f_cptr (f, c->next);
     }
 
   return n_chunks;
@@ -1558,7 +1569,7 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
 
   placeholder_fifo = svm_fifo_alloc (f->shr->size);
   svm_fifo_init (f, f->shr->size);
-  clib_memset (f->shr->head_chunk->data, 0xFF, f->shr->size);
+  clib_memset (f_head_cptr (f)->data, 0xFF, f->shr->size);
   vec_validate (data, f->shr->size);
   for (i = 0; i < vec_len (data); i++)
     data[i] = i;

diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 5845d70..f5b6080 100644 (file)
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -171,6 +171,48 @@ f_chunk_includes_pos (svm_fifo_chunk_t * c, u32 pos)
          && f_pos_lt (pos, c->start_byte + c->length));
 }
 
+always_inline svm_fifo_chunk_t *
+f_start_cptr (svm_fifo_t *f)
+{
+  return fs_chunk_ptr (f->fs_hdr, f->shr->start_chunk);
+}
+
+always_inline svm_fifo_chunk_t *
+f_end_cptr (svm_fifo_t *f)
+{
+  return fs_chunk_ptr (f->fs_hdr, f->shr->end_chunk);
+}
+
+always_inline svm_fifo_chunk_t *
+f_head_cptr (svm_fifo_t *f)
+{
+  return fs_chunk_ptr (f->fs_hdr, f->shr->head_chunk);
+}
+
+always_inline svm_fifo_chunk_t *
+f_tail_cptr (svm_fifo_t *f)
+{
+  return fs_chunk_ptr (f->fs_hdr, f->shr->tail_chunk);
+}
+
+always_inline svm_fifo_chunk_t *
+f_cptr (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp)
+{
+  return fs_chunk_ptr (f->fs_hdr, cp);
+}
+
+always_inline svm_fifo_chunk_ptr_t
+f_csptr (svm_fifo_t *f, svm_fifo_chunk_t *c)
+{
+  return fs_chunk_sptr (f->fs_hdr, c);
+}
+
+always_inline void
+f_csptr_link (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp, svm_fifo_chunk_t *c)
+{
+  fs_chunk_ptr (f->fs_hdr, cp)->next = fs_chunk_sptr (f->fs_hdr, c);
+}
+
 /**
  * Create fifo of requested size
  *
@@ -606,7 +648,7 @@ u32 svm_fifo_max_write_chunk (svm_fifo_t * f);
 static inline svm_fifo_chunk_t *
 svm_fifo_head_chunk (svm_fifo_t * f)
 {
-  return f->shr->head_chunk;
+  return f_head_cptr (f);
 }
 
 /**
@@ -618,11 +660,12 @@ svm_fifo_head_chunk (svm_fifo_t * f)
 static inline u8 *
 svm_fifo_head (svm_fifo_t * f)
 {
+  svm_fifo_chunk_t *head_chunk;
   if (!f->shr->head_chunk)
     return 0;
   /* load-relaxed: consumer owned index */
-  return (f->shr->head_chunk->data +
-         (f->shr->head - f->shr->head_chunk->start_byte));
+  head_chunk = f_head_cptr (f);
+  return (head_chunk->data + (f->shr->head - head_chunk->start_byte));
 }
 
 /**
@@ -634,7 +677,7 @@ svm_fifo_head (svm_fifo_t * f)
 static inline svm_fifo_chunk_t *
 svm_fifo_tail_chunk (svm_fifo_t * f)
 {
-  return f->shr->tail_chunk;
+  return f_tail_cptr (f);
 }
 
 /**
@@ -646,9 +689,11 @@ svm_fifo_tail_chunk (svm_fifo_t * f)
 static inline u8 *
 svm_fifo_tail (svm_fifo_t * f)
 {
+  svm_fifo_chunk_t *tail_chunk;
+
   /* load-relaxed: producer owned index */
-  return (f->shr->tail_chunk->data +
-         (f->shr->tail - f->shr->tail_chunk->start_byte));
+  tail_chunk = f_tail_cptr (f);
+  return (tail_chunk->data + (f->shr->tail - tail_chunk->start_byte));
 }
 
 /**