Code Review
/
vpp.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
Remove c-11 memcpy checks from perf-critical code
[vpp.git]
/
src
/
vnet
/
classify
/
vnet_classify.c
diff --git
a/src/vnet/classify/vnet_classify.c
b/src/vnet/classify/vnet_classify.c
index
ef45d1a
..
4e82222
100644
(file)
--- a/
src/vnet/classify/vnet_classify.c
+++ b/
src/vnet/classify/vnet_classify.c
@@
-13,7
+13,7
@@
* limitations under the License.
*/
#include <vnet/classify/vnet_classify.h>
* limitations under the License.
*/
#include <vnet/classify/vnet_classify.h>
-#include <vnet/classify/in
p
ut_acl.h>
+#include <vnet/classify/in
_o
ut_acl.h>
#include <vnet/ip/ip.h>
#include <vnet/api_errno.h> /* for API error numbers */
#include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
#include <vnet/ip/ip.h>
#include <vnet/api_errno.h> /* for API error numbers */
#include <vnet/l2/l2_classify.h> /* for L2_INPUT_CLASSIFY_NEXT_xxx */
@@
-128,10
+128,10
@@
vnet_classify_new_table (vnet_classify_main_t * cm,
nbuckets = 1 << (max_log2 (nbuckets));
pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
nbuckets = 1 << (max_log2 (nbuckets));
pool_get_aligned (cm->tables, t, CLIB_CACHE_LINE_BYTES);
- memset (t, 0, sizeof (*t));
+
clib_
memset (t, 0, sizeof (*t));
vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4));
- clib_memcpy (t->mask, mask, match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(t->mask, mask, match_n_vectors * sizeof (u32x4));
t->next_table_index = ~0;
t->nbuckets = nbuckets;
t->next_table_index = ~0;
t->nbuckets = nbuckets;
@@
-140,7
+140,11
@@
vnet_classify_new_table (vnet_classify_main_t * cm,
t->skip_n_vectors = skip_n_vectors;
t->entries_per_page = 2;
t->skip_n_vectors = skip_n_vectors;
t->entries_per_page = 2;
+#if USE_DLMALLOC == 0
t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
t->mheap = mheap_alloc (0 /* use VM */ , memory_size);
+#else
+ t->mheap = create_mspace (memory_size, 1 /* locked */ );
+#endif
vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
oldheap = clib_mem_set_heap (t->mheap);
vec_validate_aligned (t->buckets, nbuckets - 1, CLIB_CACHE_LINE_BYTES);
oldheap = clib_mem_set_heap (t->mheap);
@@
-170,7
+174,11
@@
vnet_classify_delete_table_index (vnet_classify_main_t * cm,
vec_free (t->mask);
vec_free (t->buckets);
vec_free (t->mask);
vec_free (t->buckets);
+#if USE_DLMALLOC == 0
mheap_free (t->mheap);
mheap_free (t->mheap);
+#else
+ destroy_mspace (t->mheap);
+#endif
pool_put (cm->tables, t);
}
pool_put (cm->tables, t);
}
@@
-203,7
+211,7
@@
vnet_classify_entry_alloc (vnet_classify_table_t * t, u32 log2_pages)
initialize:
ASSERT (rv);
initialize:
ASSERT (rv);
- memset (rv, 0xff, required_length);
+
clib_
memset (rv, 0xff, required_length);
return rv;
}
return rv;
}
@@
-265,7
+273,7
@@
static inline void make_working_copy
v = vnet_classify_get_entry (t, b->offset);
v = vnet_classify_get_entry (t, b->offset);
- clib_memcpy (working_copy, v, required_length);
+ clib_memcpy
_fast
(working_copy, v, required_length);
working_bucket.as_u64 = b->as_u64;
working_bucket.offset = vnet_classify_get_offset (t, working_copy);
working_bucket.as_u64 = b->as_u64;
working_bucket.offset = vnet_classify_get_offset (t, working_copy);
@@
-309,8
+317,8
@@
split_and_rehash (vnet_classify_table_t * t,
if (vnet_classify_entry_is_free (new_v))
{
if (vnet_classify_entry_is_free (new_v))
{
- clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
- + (t->match_n_vectors * sizeof (u32x4)));
+ clib_memcpy
_fast
(new_v, v, sizeof (vnet_classify_entry_t)
+
+ (t->match_n_vectors * sizeof (u32x4)));
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
goto doublebreak;
}
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
goto doublebreak;
}
@@
-353,8
+361,8
@@
split_and_rehash_linear (vnet_classify_table_t * t,
clib_warning ("BUG: linear rehash new entry not free!");
continue;
}
clib_warning ("BUG: linear rehash new entry not free!");
continue;
}
- clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t)
- + (t->match_n_vectors * sizeof (u32x4)));
+ clib_memcpy
_fast
(new_v, v, sizeof (vnet_classify_entry_t)
+
+ (t->match_n_vectors * sizeof (u32x4)));
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
j++;
goto doublebreak;
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
j++;
goto doublebreak;
@@
-436,7
+444,7
@@
vnet_classify_add_del (vnet_classify_table_t * t,
hash >>= t->log2_nbuckets;
hash >>= t->log2_nbuckets;
- while (
__sync_lock_test_and_set (t->writer_lock, 1
))
+ while (
clib_atomic_test_and_set (t->writer_lock
))
;
/* First elt in the bucket? */
;
/* First elt in the bucket? */
@@
-449,8
+457,8
@@
vnet_classify_add_del (vnet_classify_table_t * t,
}
v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
}
v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ );
- clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
- t->match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(v, add_v, sizeof (vnet_classify_entry_t) +
+
t->match_n_vectors * sizeof (u32x4));
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
@@
-488,8
+496,8
@@
vnet_classify_add_del (vnet_classify_table_t * t,
if (!memcmp
(v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
{
if (!memcmp
(v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
{
- clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
- t->match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(v, add_v, sizeof (vnet_classify_entry_t) +
+
t->match_n_vectors * sizeof (u32x4));
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
@@
-505,8
+513,8
@@
vnet_classify_add_del (vnet_classify_table_t * t,
if (vnet_classify_entry_is_free (v))
{
if (vnet_classify_entry_is_free (v))
{
- clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) +
- t->match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(v, add_v, sizeof (vnet_classify_entry_t) +
+
t->match_n_vectors * sizeof (u32x4));
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (v);
@@
-528,8
+536,8
@@
vnet_classify_add_del (vnet_classify_table_t * t,
(v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
{
vnet_classify_entry_release_resource (v);
(v->key, add_v->key, t->match_n_vectors * sizeof (u32x4)))
{
vnet_classify_entry_release_resource (v);
- memset (v, 0xff, sizeof (vnet_classify_entry_t) +
- t->match_n_vectors * sizeof (u32x4));
+
clib_
memset (v, 0xff, sizeof (vnet_classify_entry_t) +
+
t->match_n_vectors * sizeof (u32x4));
v->flags |= VNET_CLASSIFY_ENTRY_FREE;
CLIB_MEMORY_BARRIER ();
v->flags |= VNET_CLASSIFY_ENTRY_FREE;
CLIB_MEMORY_BARRIER ();
@@
-601,8
+609,8
@@
vnet_classify_add_del (vnet_classify_table_t * t,
if (vnet_classify_entry_is_free (new_v))
{
if (vnet_classify_entry_is_free (new_v))
{
- clib_memcpy (new_v, add_v, sizeof (vnet_classify_entry_t) +
- t->match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(new_v, add_v, sizeof (vnet_classify_entry_t) +
+
t->match_n_vectors * sizeof (u32x4));
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (new_v);
new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE);
vnet_classify_entry_claim_resource (new_v);
@@
-837,7
+845,7
@@
unformat_tcp_mask (unformat_input_t * input, va_list * args)
tcp = (tcp_header_t *) mask;
tcp = (tcp_header_t *) mask;
-#define _(a) if (a) memset (&tcp->a, 0xff, sizeof (tcp->a));
+#define _(a) if (a)
clib_
memset (&tcp->a, 0xff, sizeof (tcp->a));
foreach_tcp_proto_field;
#undef _
foreach_tcp_proto_field;
#undef _
@@
-878,7
+886,7
@@
unformat_udp_mask (unformat_input_t * input, va_list * args)
udp = (udp_header_t *) mask;
udp = (udp_header_t *) mask;
-#define _(a) if (a) memset (&udp->a, 0xff, sizeof (udp->a));
+#define _(a) if (a)
clib_
memset (&udp->a, 0xff, sizeof (udp->a));
foreach_udp_proto_field;
#undef _
foreach_udp_proto_field;
#undef _
@@
-973,7
+981,7
@@
unformat_ip4_mask (unformat_input_t * input, va_list * args)
ip = (ip4_header_t *) mask;
ip = (ip4_header_t *) mask;
-#define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
+#define _(a) if (a)
clib_
memset (&ip->a, 0xff, sizeof (ip->a));
foreach_ip4_proto_field;
#undef _
foreach_ip4_proto_field;
#undef _
@@
-1045,7
+1053,7
@@
unformat_ip6_mask (unformat_input_t * input, va_list * args)
ip = (ip6_header_t *) mask;
ip = (ip6_header_t *) mask;
-#define _(a) if (a) memset (&ip->a, 0xff, sizeof (ip->a));
+#define _(a) if (a)
clib_
memset (&ip->a, 0xff, sizeof (ip->a));
foreach_ip6_proto_field;
#undef _
foreach_ip6_proto_field;
#undef _
@@
-1141,10
+1149,10
@@
unformat_l2_mask (unformat_input_t * input, va_list * args)
vec_validate (mask, len - 1);
if (dst)
vec_validate (mask, len - 1);
if (dst)
- memset (mask, 0xff, 6);
+
clib_
memset (mask, 0xff, 6);
if (src)
if (src)
- memset (mask + 6, 0xff, 6);
+
clib_
memset (mask + 6, 0xff, 6);
if (tag2 || dot1ad)
{
if (tag2 || dot1ad)
{
@@
-1877,10
+1885,10
@@
unformat_ip6_match (unformat_input_t * input, va_list * args)
ip = (ip6_header_t *) match;
if (src)
ip = (ip6_header_t *) match;
if (src)
- clib_memcpy (&ip->src_address, &src_val, sizeof (ip->src_address));
+ clib_memcpy
_fast
(&ip->src_address, &src_val, sizeof (ip->src_address));
if (dst)
if (dst)
- clib_memcpy (&ip->dst_address, &dst_val, sizeof (ip->dst_address));
+ clib_memcpy
_fast
(&ip->dst_address, &dst_val, sizeof (ip->dst_address));
if (proto)
ip->protocol = proto_val;
if (proto)
ip->protocol = proto_val;
@@
-2004,10
+2012,10
@@
unformat_l2_match (unformat_input_t * input, va_list * args)
vec_validate_aligned (match, len - 1, sizeof (u32x4));
if (dst)
vec_validate_aligned (match, len - 1, sizeof (u32x4));
if (dst)
- clib_memcpy (match, dst_val, 6);
+ clib_memcpy
_fast
(match, dst_val, 6);
if (src)
if (src)
- clib_memcpy (match + 6, src_val, 6);
+ clib_memcpy
_fast
(match + 6, src_val, 6);
if (tag2)
{
if (tag2)
{
@@
-2180,8
+2188,8
@@
vnet_classify_add_del_session (vnet_classify_main_t * cm,
e->metadata = 0;
/* Copy key data, honoring skip_n_vectors */
e->metadata = 0;
/* Copy key data, honoring skip_n_vectors */
- clib_memcpy (&e->key, match + t->skip_n_vectors * sizeof (u32x4),
- t->match_n_vectors * sizeof (u32x4));
+ clib_memcpy
_fast
(&e->key, match + t->skip_n_vectors * sizeof (u32x4),
+
t->match_n_vectors * sizeof (u32x4));
/* Clear don't-care bits; likely when dynamically creating sessions */
for (i = 0; i < t->match_n_vectors; i++)
/* Clear don't-care bits; likely when dynamically creating sessions */
for (i = 0; i < t->match_n_vectors; i++)
@@
-2493,7
+2501,7
@@
test_classify_churn (test_classify_main_t * tm)
data = (classify_data_or_mask_t *) dp;
/* Mask on src address */
data = (classify_data_or_mask_t *) dp;
/* Mask on src address */
- memset (&mask->ip.src_address, 0xff, 4);
+
clib_
memset (&mask->ip.src_address, 0xff, 4);
tmp = clib_host_to_net_u32 (tm->src.as_u32);
tmp = clib_host_to_net_u32 (tm->src.as_u32);