Remove c-11 memcpy checks from perf-critical code

[vpp.git] / src / vnet / classify / vnet_classify.c

diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c
index ef45d1a..4e82222 100644 (file)
--- a/src/vnet/classify/vnet_classify.c
+++ b/src/vnet/classify/vnet_classify.c
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 #include <vnet/classify/vnet_classify.h>
-#include <vnet/classify/input_acl.h>
+#include <vnet/classify/in_out_acl.h>
 #include <vnet/ip/ip.h>
 #include <vnet/api_errno.h>    /* for API error numbers */
 #include <vnet/l2/l2_classify.h>       /* for L2_INPUT_CLASSIFY_NEXT_xxx */
@@ -128,10 +128,10 @@ vnet_classify_new_table (vnet_classify_main_t * cm,
   nbuckets = 1 << (max_log2 (nbuckets));
 
   pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
-  memset (t, 0, sizeof (*t));
+  clib_memset (t, 0, sizeof (*t));
 
   vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
-  clib_memcpy (t->mask, mask, match_n_vectors * sizeof (u32x4));
+  clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4));
 
   t->next_table_index = ~0;
   t->nbuckets = nbuckets;
@@ -140,7 +140,11 @@ vnet_classify_new_table (vnet_classify_main_t * cm,
   t->skip_n_vectors = skip_n_vectors;
   t->entries_per_page = 2;
 
+#if USE_DLMALLOC == 0
   t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
+#else
+  t->mheap = create_mspace (memory_size, 1 /* locked */ );
+#endif
 
   vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
   oldheap = clib_mem_set_heap (t->mheap);
@@ -170,7 +174,11 @@ vnet_classify_delete_table_index (vnet_classify_main_t * cm,
 
   vec_free (t->mask);
   vec_free (t->buckets);
+#if USE_DLMALLOC == 0
   mheap_free (t->mheap);
+#else
+  destroy_mspace (t->mheap);
+#endif
 
   pool_put (cm->tables, t);
 }
@@ -203,7 +211,7 @@ vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
 initialize:
   ASSERT (rv);
 
-  memset (rv, 0xff, required_length);
+  clib_memset (rv, 0xff, required_length);
   return rv;
 }
 
@@ -265,7 +273,7 @@ static inline void make_working_copy
 
   v = vnet_classify_get_entry (t, b->offset);
 
-  clib_memcpy (working_copy, v, required_length);
+  clib_memcpy_fast (working_copy, v, required_length);
 
   working_bucket.as_u64 = b->as_u64;
   working_bucket.offset = vnet_classify_get_offset (t, working_copy);
@@ -309,8 +317,8 @@ split_and_rehash (vnet_classify_table_t * t,
 
              if (vnet_classify_entry_is_free (new_v))
                {
-                 clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
-                              + (t->match_n_vectors * sizeof (u32x4)));
+                 clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
+                                   + (t->match_n_vectors * sizeof (u32x4)));
                  new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
                  goto doublebreak;
                }
@@ -353,8 +361,8 @@ split_and_rehash_linear (vnet_classify_table_t * t,
                  clib_warning ("BUG: linear rehash new entry not free!");
                  continue;
                }
-             clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
-                          + (t->match_n_vectors * sizeof (u32x4)));
+             clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t)
+                               + (t->match_n_vectors * sizeof (u32x4)));
              new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
              j++;
              goto doublebreak;
@@ -436,7 +444,7 @@ vnet_classify_add_del (vnet_classify_table_t * t,
 
   hash >>= t->log2_nbuckets;
 
-  while (__sync_lock_test_and_set (t->writer_lock, 1))
+  while (clib_atomic_test_and_set (t->writer_lock))
     ;
 
   /* First elt in the bucket? */
@@ -449,8 +457,8 @@ vnet_classify_add_del (vnet_classify_table_t * t,
        }
 
       v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
-      clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
-                  t->match_n_vectors * sizeof (u32x4));
+      clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
+                       t->match_n_vectors * sizeof (u32x4));
       v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
       vnet_classify_entry_claim_resource (v);
 
@@ -488,8 +496,8 @@ vnet_classify_add_del (vnet_classify_table_t * t,
          if (!memcmp
              (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
            {
-             clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
-                          t->match_n_vectors * sizeof (u32x4));
+             clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
+                               t->match_n_vectors * sizeof (u32x4));
              v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
              vnet_classify_entry_claim_resource (v);
 
@@ -505,8 +513,8 @@ vnet_classify_add_del (vnet_classify_table_t * t,
 
          if (vnet_classify_entry_is_free (v))
            {
-             clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
-                          t->match_n_vectors * sizeof (u32x4));
+             clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) +
+                               t->match_n_vectors * sizeof (u32x4));
              v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
              vnet_classify_entry_claim_resource (v);
 
@@ -528,8 +536,8 @@ vnet_classify_add_del (vnet_classify_table_t * t,
              (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
            {
              vnet_classify_entry_release_resource (v);
-             memset (v, 0xff, sizeof (vnet_classify_entry_t) +
-                     t->match_n_vectors * sizeof (u32x4));
+             clib_memset (v, 0xff, sizeof (vnet_classify_entry_t) +
+                          t->match_n_vectors * sizeof (u32x4));
              v->flags |= VNET_CLASSIFY_ENTRY_FREE;
 
              CLIB_MEMORY_BARRIER ();
@@ -601,8 +609,8 @@ vnet_classify_add_del (vnet_classify_table_t * t,
 
       if (vnet_classify_entry_is_free (new_v))
        {
-         clib_memcpy (new_v, add_v, sizeof (vnet_classify_entry_t) +
-                      t->match_n_vectors * sizeof (u32x4));
+         clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) +
+                           t->match_n_vectors * sizeof (u32x4));
          new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
          vnet_classify_entry_claim_resource (new_v);
 
@@ -837,7 +845,7 @@ unformat_tcp_mask (unformat_input_t * input, va_list * args)
 
   tcp = (tcp_header_t *) mask;
 
-#define _(a) if (a) memset (&tcp->a, 0xff, sizeof (tcp->a));
+#define _(a) if (a) clib_memset (&tcp->a, 0xff, sizeof (tcp->a));
   foreach_tcp_proto_field;
 #undef _
 
@@ -878,7 +886,7 @@ unformat_udp_mask (unformat_input_t * input, va_list * args)
 
   udp = (udp_header_t *) mask;
 
-#define _(a) if (a) memset (&udp->a, 0xff, sizeof (udp->a));
+#define _(a) if (a) clib_memset (&udp->a, 0xff, sizeof (udp->a));
   foreach_udp_proto_field;
 #undef _
 
@@ -973,7 +981,7 @@ unformat_ip4_mask (unformat_input_t * input, va_list * args)
 
   ip = (ip4_header_t *) mask;
 
-#define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
+#define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
   foreach_ip4_proto_field;
 #undef _
 
@@ -1045,7 +1053,7 @@ unformat_ip6_mask (unformat_input_t * input, va_list * args)
 
   ip = (ip6_header_t *) mask;
 
-#define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
+#define _(a) if (a) clib_memset (&ip->a, 0xff, sizeof (ip->a));
   foreach_ip6_proto_field;
 #undef _
 
@@ -1141,10 +1149,10 @@ unformat_l2_mask (unformat_input_t * input, va_list * args)
   vec_validate (mask, len - 1);
 
   if (dst)
-    memset (mask, 0xff, 6);
+    clib_memset (mask, 0xff, 6);
 
   if (src)
-    memset (mask + 6, 0xff, 6);
+    clib_memset (mask + 6, 0xff, 6);
 
   if (tag2 || dot1ad)
     {
@@ -1877,10 +1885,10 @@ unformat_ip6_match (unformat_input_t * input, va_list * args)
   ip = (ip6_header_t *) match;
 
   if (src)
-    clib_memcpy (&ip->src_address, &src_val, sizeof (ip->src_address));
+    clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address));
 
   if (dst)
-    clib_memcpy (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
+    clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
 
   if (proto)
     ip->protocol = proto_val;
@@ -2004,10 +2012,10 @@ unformat_l2_match (unformat_input_t * input, va_list * args)
   vec_validate_aligned (match, len - 1, sizeof (u32x4));
 
   if (dst)
-    clib_memcpy (match, dst_val, 6);
+    clib_memcpy_fast (match, dst_val, 6);
 
   if (src)
-    clib_memcpy (match + 6, src_val, 6);
+    clib_memcpy_fast (match + 6, src_val, 6);
 
   if (tag2)
     {
@@ -2180,8 +2188,8 @@ vnet_classify_add_del_session (vnet_classify_main_t * cm,
     e->metadata = 0;
 
   /* Copy key data, honoring skip_n_vectors */
-  clib_memcpy (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
-              t->match_n_vectors * sizeof (u32x4));
+  clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
+                   t->match_n_vectors * sizeof (u32x4));
 
   /* Clear don't-care bits; likely when dynamically creating sessions */
   for (i = 0; i < t->match_n_vectors; i++)
@@ -2493,7 +2501,7 @@ test_classify_churn (test_classify_main_t * tm)
   data = (classify_data_or_mask_t *) dp;
 
   /* Mask on src address */
-  memset (&mask->ip.src_address, 0xff, 4);
+  clib_memset (&mask->ip.src_address, 0xff, 4);
 
   tmp = clib_host_to_net_u32 (tm->src.as_u32);