vppinfra: numa vector placement support

[vpp.git] / src / vppinfra / vec.h

diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h
index de90abf..9054eaa 100644 (file)
--- a/src/vppinfra/vec.h
+++ b/src/vppinfra/vec.h
@@ -70,16 +70,18 @@
    Typically, the header is not present.  Headers allow for other
    data structures to be built atop CLIB vectors.
 
    Typically, the header is not present.  Headers allow for other
    data structures to be built atop CLIB vectors.
 

-   Users may specify the alignment for data elements via the
-   vec_*_aligned macros.
+   Users may specify the alignment for first data element of a vector
+   via the vec_*_aligned macros.

 
 

-   Vectors elements can be any C type e.g. (int, double, struct bar).
+   Vector elements can be any C type e.g. (int, double, struct bar).

    This is also true for data types built atop vectors (e.g. heap,
    pool, etc.).
 
    This is also true for data types built atop vectors (e.g. heap,
    pool, etc.).
 

-   Many macros have _a variants supporting alignment of vector data
-   and _h variants supporting non zero length vector headers.
-   The _ha variants support both.
+   Many macros have \_a variants supporting alignment of vector elements
+   and \_h variants supporting non-zero-length vector headers. The \_ha
+   variants support both.  Additionally cacheline alignment within a
+   vector element structure can be specified using the
+   CLIB_CACHE_LINE_ALIGN_MARK() macro.

 
    Standard programming error: memorize a pointer to the ith element
    of a vector then expand it. Vectors expand by 3/2, so such code
 
    Standard programming error: memorize a pointer to the ith element
    of a vector then expand it. Vectors expand by 3/2, so such code


@@ -94,12 +96,14 @@
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)

+    @param numa_id numa id (may be zero)

     @return v_prime pointer to resized vector, may or may not equal v
 */
 void *vec_resize_allocate_memory (void *v,
                                  word length_increment,
                                  uword data_bytes,
     @return v_prime pointer to resized vector, may or may not equal v
 */
 void *vec_resize_allocate_memory (void *v,
                                  word length_increment,
                                  uword data_bytes,

-                                 uword header_bytes, uword data_align);
+                                 uword header_bytes, uword data_align,
+                                 uword numa_id);

 
 /** \brief Low-level vector resize function, usually not called directly
 
 
 /** \brief Low-level vector resize function, usually not called directly
 


@@ -108,16 +112,25 @@ void *vec_resize_allocate_memory (void *v,
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)

+    @param numa_id (may be ~0)

     @return v_prime pointer to resized vector, may or may not equal v
 */
 
     @return v_prime pointer to resized vector, may or may not equal v
 */
 

+#define _vec_resize_numa(V,L,DB,HB,A,S) \
+  _vec_resize_inline(V,L,DB,HB,clib_max((__alignof__((V)[0])),(A)),(S))
+
+#define _vec_resize(V,L,DB,HB,A)  \
+  _vec_resize_numa(V,L,DB,HB,A,VEC_NUMA_UNSPECIFIED)
+

 always_inline void *
 always_inline void *

-_vec_resize (void *v,
-            word length_increment,
-            uword data_bytes, uword header_bytes, uword data_align)
+_vec_resize_inline (void *v,
+                   word length_increment,
+                   uword data_bytes, uword header_bytes, uword data_align,
+                   uword numa_id)

 {
   vec_header_t *vh = _vec_find (v);
   uword new_data_bytes, aligned_header_bytes;
 {
   vec_header_t *vh = _vec_find (v);
   uword new_data_bytes, aligned_header_bytes;

+  void *oldheap;

 
   aligned_header_bytes = vec_header_bytes (header_bytes);
 
 
   aligned_header_bytes = vec_header_bytes (header_bytes);
 


@@ -127,32 +140,43 @@ _vec_resize (void *v,
     {
       void *p = v - aligned_header_bytes;
 
     {
       void *p = v - aligned_header_bytes;
 

+      if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED))
+       {
+         oldheap = clib_mem_get_per_cpu_heap ();
+         clib_mem_set_per_cpu_heap (clib_mem_get_per_numa_heap (numa_id));
+       }
+

       /* Vector header must start heap object. */
       ASSERT (clib_mem_is_heap_object (p));
 
       /* Typically we'll not need to resize. */
       if (new_data_bytes <= clib_mem_size (p))
        {
       /* Vector header must start heap object. */
       ASSERT (clib_mem_is_heap_object (p));
 
       /* Typically we'll not need to resize. */
       if (new_data_bytes <= clib_mem_size (p))
        {

+         CLIB_MEM_UNPOISON (v, data_bytes);

          vh->len += length_increment;
          vh->len += length_increment;

+         if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED))
+           clib_mem_set_per_cpu_heap (oldheap);

          return v;
        }
          return v;
        }

+      if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED))
+       clib_mem_set_per_cpu_heap (oldheap);

     }
 
   /* Slow path: call helper function. */
   return vec_resize_allocate_memory (v, length_increment, data_bytes,
                                     header_bytes,
                                     clib_max (sizeof (vec_header_t),
     }
 
   /* Slow path: call helper function. */
   return vec_resize_allocate_memory (v, length_increment, data_bytes,
                                     header_bytes,
                                     clib_max (sizeof (vec_header_t),

-                                              data_align));
+                                              data_align), numa_id);

 }
 
 }
 

-/** \brief Low-level vector resize predicate
+/** \brief Determine if vector will resize with next allocation

 
     @param v pointer to a vector
     @param length_increment length increment in elements
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)
 
     @param v pointer to a vector
     @param length_increment length increment in elements
     @param data_bytes requested size in bytes
     @param header_bytes header size in bytes (may be zero)
     @param data_align alignment (may be zero)

-    @return v_prime pointer to resized vector, may or may not equal v
+    @return 1 if vector will resize 0 otherwise

 */
 
 always_inline int
 */
 
 always_inline int


@@ -161,7 +185,6 @@ _vec_resize_will_expand (void *v,
                         uword data_bytes, uword header_bytes,
                         uword data_align)
 {
                         uword data_bytes, uword header_bytes,
                         uword data_align)
 {

-  vec_header_t *vh = _vec_find (v);

   uword new_data_bytes, aligned_header_bytes;
 
   aligned_header_bytes = vec_header_bytes (header_bytes);
   uword new_data_bytes, aligned_header_bytes;
 
   aligned_header_bytes = vec_header_bytes (header_bytes);


@@ -177,10 +200,7 @@ _vec_resize_will_expand (void *v,
 
       /* Typically we'll not need to resize. */
       if (new_data_bytes <= clib_mem_size (p))
 
       /* Typically we'll not need to resize. */
       if (new_data_bytes <= clib_mem_size (p))

-       {
-         vh->len += length_increment;
-         return 0;
-       }
+       return 0;

     }
   return 1;
 }
     }
   return 1;
 }


@@ -219,16 +239,32 @@ clib_mem_is_vec (void *v)
     @param N number of elements to add
     @param H header size in bytes (may be zero)
     @param A alignment (may be zero)
     @param N number of elements to add
     @param H header size in bytes (may be zero)
     @param A alignment (may be zero)

+    @param S numa_id (may be zero)

     @return V (value-result macro parameter)
 */
 
     @return V (value-result macro parameter)
 */
 

-#define vec_resize_ha(V,N,H,A)                                                 \
-do {                                                                           \
-  word _v(n) = (N);                                                            \
-  word _v(l) = vec_len (V);                                                    \
-  V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A));   \
+#define vec_resize_has(V,N,H,A,S)                               \
+do {                                                            \
+  word _v(n) = (N);                                             \
+  word _v(l) = vec_len (V);                                     \
+  V = _vec_resize_numa ((V), _v(n),                           \
+                          (_v(l) + _v(n)) * sizeof ((V)[0]),    \
+                          (H), (A),(S));                        \

 } while (0)
 
 } while (0)
 

+/** \brief Resize a vector (less general version).
+   Add N elements to end of given vector V, return pointer to start of vector.
+   Vector will have room for H header bytes and will have user's data aligned
+   at alignment A (rounded to next power of 2).
+
+    @param V pointer to a vector
+    @param N number of elements to add
+    @param H header size in bytes (may be zero)
+    @param A alignment (may be zero)
+    @return V (value-result macro parameter)
+*/
+#define vec_resize_ha(V,N,H,A) vec_resize_has(V,N,H,A,VEC_NUMA_UNSPECIFIED)
+

 /** \brief Resize a vector (no header, unspecified alignment)
    Add N elements to end of given vector V, return pointer to start of vector.
    Vector will have room for H header bytes and will have user's data aligned
 /** \brief Resize a vector (no header, unspecified alignment)
    Add N elements to end of given vector V, return pointer to start of vector.
    Vector will have room for H header bytes and will have user's data aligned


@@ -350,22 +386,35 @@ do {                                              \
     @param V pointer to a vector
     @param H size of header in bytes
     @param A alignment (may be zero)
     @param V pointer to a vector
     @param H size of header in bytes
     @param A alignment (may be zero)

+    @param S numa (may be VEC_NUMA_UNSPECIFIED)

 
     @return Vdup copy of vector
 */
 
 
     @return Vdup copy of vector
 */
 

-#define vec_dup_ha(V,H,A)                              \
+#define vec_dup_ha_numa(V,H,A,S)                      \

 ({                                                     \
   __typeof__ ((V)[0]) * _v(v) = 0;                     \
   uword _v(l) = vec_len (V);                           \
   if (_v(l) > 0)                                       \
     {                                                  \
 ({                                                     \
   __typeof__ ((V)[0]) * _v(v) = 0;                     \
   uword _v(l) = vec_len (V);                           \
   if (_v(l) > 0)                                       \
     {                                                  \

-      vec_resize_ha (_v(v), _v(l), (H), (A));          \
-      clib_memcpy (_v(v), (V), _v(l) * sizeof ((V)[0]));\
+      vec_resize_has (_v(v), _v(l), (H), (A), (S));     \
+      clib_memcpy_fast (_v(v), (V), _v(l) * sizeof ((V)[0]));\

     }                                                  \
   _v(v);                                               \
 })
 
     }                                                  \
   _v(v);                                               \
 })
 

+/** \brief Return copy of vector (VEC_NUMA_UNSPECIFIED).
+
+    @param V pointer to a vector
+    @param H size of header in bytes
+    @param A alignment (may be zero)
+
+    @return Vdup copy of vector
+*/
+#define vec_dup_ha(V,H,A) \
+  vec_dup_ha_numa(V,H,A,VEC_NUMA_UNSPECIFIED)
+
+

 /** \brief Return copy of vector (no header, no alignment)
 
     @param V pointer to a vector
 /** \brief Return copy of vector (no header, no alignment)
 
     @param V pointer to a vector


@@ -388,7 +437,7 @@ do {                                                \
     @param DST destination
     @param SRC source
 */
     @param DST destination
     @param SRC source
 */

-#define vec_copy(DST,SRC) clib_memcpy (DST, SRC, vec_len (DST) * \
+#define vec_copy(DST,SRC) clib_memcpy_fast (DST, SRC, vec_len (DST) * \

                                       sizeof ((DST)[0]))
 
 /** \brief Clone a vector. Make a new vector with the
                                       sizeof ((DST)[0]))
 
 /** \brief Clone a vector. Make a new vector with the


@@ -410,22 +459,40 @@ do {                                                                              \
     @param I vector index which will be valid upon return
     @param H header size in bytes (may be zero)
     @param A alignment (may be zero)
     @param I vector index which will be valid upon return
     @param H header size in bytes (may be zero)
     @param A alignment (may be zero)

+    @param N numa_id (may be zero)

     @return V (value-result macro parameter)
 */
 
     @return V (value-result macro parameter)
 */
 

-#define vec_validate_ha(V,I,H,A)                                       \
-do {                                                                   \
-  word _v(i) = (I);                                                    \
-  word _v(l) = vec_len (V);                                            \
-  if (_v(i) >= _v(l))                                                  \
-    {                                                                  \
-      vec_resize_ha ((V), 1 + (_v(i) - _v(l)), (H), (A));              \
-      /* Must zero new space since user may have previously            \
-        used e.g. _vec_len (v) -= 10 */                                \
-      memset ((V) + _v(l), 0, (1 + (_v(i) - _v(l))) * sizeof ((V)[0]));        \
-    }                                                                  \
+#define vec_validate_han(V,I,H,A,N)                                     \
+do {                                                                    \
+  void *oldheap;                                                        \
+  STATIC_ASSERT(A==0 || ((A % sizeof(V[0]))==0)                         \
+        || ((sizeof(V[0]) % A) == 0),                                   \
+    "vector validate aligned on incorrectly sized object");             \
+  word _v(i) = (I);                                                     \
+  word _v(l) = vec_len (V);                                             \
+  if (_v(i) >= _v(l))                                                   \
+    {                                                                   \
+      /* switch to the per-numa heap if directed */                   \
+      if (PREDICT_FALSE(N != VEC_NUMA_UNSPECIFIED))                   \
+        {                                                               \
+           oldheap = clib_mem_get_per_cpu_heap();                       \
+           clib_mem_set_per_cpu_heap (clib_mem_get_per_numa_heap(N)); \
+        }                                                               \
+                                                                        \
+      vec_resize_ha ((V), 1 + (_v(i) - _v(l)), (H), (A));               \
+      /* Must zero new space since user may have previously             \
+        used e.g. _vec_len (v) -= 10 */                                \
+      clib_memset ((V) + _v(l), 0,                                      \
+                   (1 + (_v(i) - _v(l))) * sizeof ((V)[0]));            \
+      /* Switch back to the global heap */                              \
+      if (PREDICT_FALSE (N != VEC_NUMA_UNSPECIFIED))                  \
+        clib_mem_set_per_cpu_heap (oldheap);                            \
+    }                                                                   \

 } while (0)
 
 } while (0)
 

+#define vec_validate_ha(V,I,H,A) vec_validate_han(V,I,H,A,VEC_NUMA_UNSPECIFIED)
+

 /** \brief Make sure vector is long enough for given index
     (no header, unspecified alignment)
 
 /** \brief Make sure vector is long enough for given index
     (no header, unspecified alignment)
 


@@ -477,8 +544,6 @@ do {                                                                \
     @param V (possibly NULL) pointer to a vector.
     @param I vector index which will be valid upon return
     @param INIT initial value (can be a complex expression!)
     @param V (possibly NULL) pointer to a vector.
     @param I vector index which will be valid upon return
     @param INIT initial value (can be a complex expression!)

-    @param H header size in bytes (may be zero)
-    @param A alignment (may be zero)

     @return V (value-result macro parameter)
 */
 
     @return V (value-result macro parameter)
 */
 


@@ -491,7 +556,6 @@ do {                                                                \
     @param V (possibly NULL) pointer to a vector.
     @param I vector index which will be valid upon return
     @param INIT initial value (can be a complex expression!)
     @param V (possibly NULL) pointer to a vector.
     @param I vector index which will be valid upon return
     @param INIT initial value (can be a complex expression!)

-    @param H header size in bytes (may be zero)

     @param A alignment (may be zero)
     @return V (value-result macro parameter)
 */
     @param A alignment (may be zero)
     @return V (value-result macro parameter)
 */


@@ -525,7 +589,6 @@ do {                                                                        \
 
     @param V pointer to a vector
     @param E element to add
 
     @param V pointer to a vector
     @param E element to add

-    @param H header size in bytes (may be zero)

     @param A alignment (may be zero)
     @return V (value-result macro parameter)
 */
     @param A alignment (may be zero)
     @return V (value-result macro parameter)
 */


@@ -586,7 +649,7 @@ do {                                                                                \
   word _v(n) = (N);                                                            \
   word _v(l) = vec_len (V);                                                    \
   V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A));   \
   word _v(n) = (N);                                                            \
   word _v(l) = vec_len (V);                                                    \
   V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A));   \

-  clib_memcpy ((V) + _v(l), (E), _v(n) * sizeof ((V)[0]));                     \
+  clib_memcpy_fast ((V) + _v(l), (E), _v(n) * sizeof ((V)[0]));                        \

 } while (0)
 
 /** \brief Add N elements to end of vector V (no header, unspecified alignment)
 } while (0)
 
 /** \brief Add N elements to end of vector V (no header, unspecified alignment)


@@ -660,7 +723,7 @@ do {                                                        \
   memmove ((V) + _v(m) + _v(n),                                \
           (V) + _v(m),                                 \
           (_v(l) - _v(m)) * sizeof ((V)[0]));          \
   memmove ((V) + _v(m) + _v(n),                                \
           (V) + _v(m),                                 \
           (_v(l) - _v(m)) * sizeof ((V)[0]));          \

-  memset  ((V) + _v(m), INIT, _v(n) * sizeof ((V)[0]));        \
+  clib_memset  ((V) + _v(m), INIT, _v(n) * sizeof ((V)[0]));   \

 } while (0)
 
 /** \brief Insert N vector elements starting at element M,
 } while (0)
 
 /** \brief Insert N vector elements starting at element M,


@@ -748,7 +811,7 @@ do {                                                        \
   memmove ((V) + _v(m) + _v(n),                                \
           (V) + _v(m),                                 \
           (_v(l) - _v(m)) * sizeof ((V)[0]));          \
   memmove ((V) + _v(m) + _v(n),                                \
           (V) + _v(m),                                 \
           (_v(l) - _v(m)) * sizeof ((V)[0]));          \

-  clib_memcpy ((V) + _v(m), (E),                       \
+  clib_memcpy_fast ((V) + _v(m), (E),                  \

               _v(n) * sizeof ((V)[0]));                \
 } while (0)
 
               _v(n) * sizeof ((V)[0]));                \
 } while (0)
 


@@ -793,8 +856,9 @@ do {                                                                \
             (_v(l) - _v(n) - _v(m)) * sizeof ((V)[0]));        \
   /* Zero empty space at end (for future re-allocation). */    \
   if (_v(n) > 0)                                               \
             (_v(l) - _v(n) - _v(m)) * sizeof ((V)[0]));        \
   /* Zero empty space at end (for future re-allocation). */    \
   if (_v(n) > 0)                                               \

-    memset ((V) + _v(l) - _v(n), 0, _v(n) * sizeof ((V)[0]));  \
+    clib_memset ((V) + _v(l) - _v(n), 0, _v(n) * sizeof ((V)[0]));     \

   _vec_len (V) -= _v(n);                                       \
   _vec_len (V) -= _v(n);                                       \

+  CLIB_MEM_POISON(vec_end(V), _v(n) * sizeof ((V)[0]));         \

 } while (0)
 
 /** \brief Delete the element at index I
 } while (0)
 
 /** \brief Delete the element at index I


@@ -809,6 +873,7 @@ do {                                                \
   if (_vec_del_i < _vec_del_l)                 \
     (v)[_vec_del_i] = (v)[_vec_del_l];         \
   _vec_len (v) = _vec_del_l;                   \
   if (_vec_del_i < _vec_del_l)                 \
     (v)[_vec_del_i] = (v)[_vec_del_l];         \
   _vec_len (v) = _vec_del_l;                   \

+  CLIB_MEM_POISON(vec_end(v), sizeof ((v)[0])); \

 } while (0)
 
 /** \brief Append v2 after v1. Result in v1.
 } while (0)
 
 /** \brief Append v2 after v1. Result in v1.


@@ -823,7 +888,7 @@ do {                                                                        \
                                                                        \
   v1 = _vec_resize ((v1), _v(l2),                                      \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0);        \
                                                                        \
   v1 = _vec_resize ((v1), _v(l2),                                      \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0);        \

-  clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0]));                \
+  clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0]));           \

 } while (0)
 
 /** \brief Append v2 after v1. Result in v1. Specified alignment.
 } while (0)
 
 /** \brief Append v2 after v1. Result in v1. Specified alignment.


@@ -839,7 +904,7 @@ do {                                                                        \
                                                                        \
   v1 = _vec_resize ((v1), _v(l2),                                      \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align);    \
                                                                        \
   v1 = _vec_resize ((v1), _v(l2),                                      \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align);    \

-  clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0]));                \
+  clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0]));           \

 } while (0)
 
 /** \brief Prepend v2 before v1. Result in v1.
 } while (0)
 
 /** \brief Prepend v2 before v1. Result in v1.


@@ -855,7 +920,7 @@ do {                                                                    \
   v1 = _vec_resize ((v1), _v(l2),                                       \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0);        \
   memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0]));             \
   v1 = _vec_resize ((v1), _v(l2),                                       \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0);        \
   memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0]));             \

-  clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0]));                  \
+  clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0]));                  \

 } while (0)
 
 /** \brief Prepend v2 before v1. Result in v1. Specified alignment
 } while (0)
 
 /** \brief Prepend v2 before v1. Result in v1. Specified alignment


@@ -872,7 +937,7 @@ do {                                                                    \
   v1 = _vec_resize ((v1), _v(l2),                                       \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align);    \
   memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0]));             \
   v1 = _vec_resize ((v1), _v(l2),                                       \
                    (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align);    \
   memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0]));             \

-  clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0]));                  \
+  clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0]));                  \

 } while (0)
 
 
 } while (0)
 
 


@@ -882,7 +947,7 @@ do {                                                                    \
 #define vec_zero(var)                                          \
 do {                                                           \
   if (var)                                                     \
 #define vec_zero(var)                                          \
 do {                                                           \
   if (var)                                                     \

-    memset ((var), 0, vec_len (var) * sizeof ((var)[0]));      \
+    clib_memset ((var), 0, vec_len (var) * sizeof ((var)[0])); \

 } while (0)
 
 /** \brief Set all vector elements to given value. Null-pointer tolerant.
 } while (0)
 
 /** \brief Set all vector elements to given value. Null-pointer tolerant.


@@ -934,8 +999,8 @@ do {                                                \
 
 /** \brief Search a vector for the index of the entry that matches.
 
 
 /** \brief Search a vector for the index of the entry that matches.
 

-    @param v1 Pointer to a vector
-    @param v2 Entry to match
+    @param v Pointer to a vector
+    @param E Entry to match

     @return index of match or ~0
 */
 #define vec_search(v,E)                                        \
     @return index of match or ~0
 */
 #define vec_search(v,E)                                        \


@@ -952,6 +1017,27 @@ do {                                              \
   _v(i);                                               \
 })
 
   _v(i);                                               \
 })
 

+/** \brief Search a vector for the index of the entry that matches.
+
+    @param v Pointer to a vector
+    @param E Pointer to entry to match
+    @param fn Comparison function !0 => match
+    @return index of match or ~0
+*/
+#define vec_search_with_function(v,E,fn)                \
+({                                                     \
+  word _v(i) = 0;                                      \
+  while (_v(i) < vec_len(v))                           \
+  {                                                    \
+    if (0 != fn(&(v)[_v(i)], (E)))                      \
+      break;                                           \
+    _v(i)++;                                           \
+  }                                                    \
+  if (_v(i) == vec_len(v))                             \
+    _v(i) = ~0;                                                \
+  _v(i);                                               \
+})
+

 /** \brief Sort a vector using the supplied element comparison function
 
     @param vec vector to sort
 /** \brief Sort a vector using the supplied element comparison function
 
     @param vec vector to sort


@@ -973,7 +1059,7 @@ do {                                                               \
     vec_reset_length (V);                       \
     vec_validate ((V), (L));                    \
     if ((S) && (L))                             \
     vec_reset_length (V);                       \
     vec_validate ((V), (L));                    \
     if ((S) && (L))                             \

-        clib_memcpy ((V), (S), (L));            \
+        clib_memcpy_fast ((V), (S), (L));            \

     (V)[(L)] = 0;                               \
   } while (0)
 
     (V)[(L)] = 0;                               \
   } while (0)