always_inline void
zero_pair (hash_t * h, hash_pair_t * p)
{
- memset (p, 0, hash_pair_bytes (h));
+ clib_memset (p, 0, hash_pair_bytes (h));
}
always_inline void
init_pair (hash_t * h, hash_pair_t * p)
{
- memset (p->value, ~0, hash_value_bytes (h));
+ clib_memset (p->value, ~0, hash_value_bytes (h));
}
always_inline hash_pair_union_t *
* Therefore all the 8 Bytes of the u64 are systematically read, which
* rightfully causes address-sanitizer to raise an error on smaller inputs.
*
- * However the invalid Bytes are discarded within zap64(), whicj is why
+ * However the invalid Bytes are discarded within zap64(), which is why
* this can be silenced safely.
+ *
+ * The above is true *unless* the extra bytes cross a page boundary
+ * into unmapped or no-access space, hence the boundary crossing check.
*/
-static inline u64 __attribute__ ((no_sanitize_address))
+static inline u64
hash_memory64 (void *p, word n_bytes, u64 state)
{
u64 *q = p;
u64 a, b, c, n;
+ int page_boundary_crossing;
+ u64 start_addr, end_addr;
+ union
+ {
+ u8 as_u8[8];
+ u64 as_u64;
+ } tmp;
+
+ /*
+ * If the request crosses a 4k boundary, it's not OK to assume
+ * that the zap64 game is safe. 4k is the minimum known page size.
+ */
+ start_addr = (u64) p;
+ end_addr = start_addr + n_bytes + 7;
+ page_boundary_crossing = (start_addr >> 12) != (end_addr >> 12);
a = b = 0x9e3779b97f4a7c13LL;
c = state;
a += clib_mem_unaligned (q + 0, u64);
b += clib_mem_unaligned (q + 1, u64);
if (n % sizeof (u64))
- c += zap64 (clib_mem_unaligned (q + 2, u64), n % sizeof (u64)) << 8;
+ {
+ if (PREDICT_TRUE (page_boundary_crossing == 0))
+ {
+ CLIB_MEM_OVERFLOW_PUSH (q + 2, sizeof (u64));
+ c += zap64 (clib_mem_unaligned (q + 2, u64), n % sizeof (u64))
+ << 8;
+ CLIB_MEM_OVERFLOW_POP ();
+ }
+ else
+ {
+ clib_memcpy_fast (tmp.as_u8, q + 2, n % sizeof (u64));
+ c += zap64 (tmp.as_u64, n % sizeof (u64)) << 8;
+ }
+ }
break;
case 1:
a += clib_mem_unaligned (q + 0, u64);
if (n % sizeof (u64))
- b += zap64 (clib_mem_unaligned (q + 1, u64), n % sizeof (u64));
+ {
+ if (PREDICT_TRUE (page_boundary_crossing == 0))
+ {
+ CLIB_MEM_OVERFLOW_PUSH (q + 1, sizeof (u64));
+ b += zap64 (clib_mem_unaligned (q + 1, u64), n % sizeof (u64));
+ CLIB_MEM_OVERFLOW_POP ();
+ }
+ else
+ {
+ clib_memcpy_fast (tmp.as_u8, q + 1, n % sizeof (u64));
+ b += zap64 (tmp.as_u64, n % sizeof (u64));
+ }
+ }
break;
case 0:
if (n % sizeof (u64))
- a += zap64 (clib_mem_unaligned (q + 0, u64), n % sizeof (u64));
+ {
+ if (PREDICT_TRUE (page_boundary_crossing == 0))
+ {
+ CLIB_MEM_OVERFLOW_PUSH (q + 0, sizeof (u64));
+ a += zap64 (clib_mem_unaligned (q + 0, u64), n % sizeof (u64));
+ CLIB_MEM_OVERFLOW_POP ();
+ }
+ else
+ {
+ clib_memcpy_fast (tmp.as_u8, q, n % sizeof (u64));
+ a += zap64 (tmp.as_u64, n % sizeof (u64));
+ }
+ }
break;
}
}
#endif
-uword
+__clib_export uword
hash_memory (void *p, word n_bytes, uword state)
{
uword *q = p;
sum = string_key_sum (h, key);
break;
+ case KEY_FUNC_MEM:
+ sum = mem_key_sum (h, key);
+ break;
+
default:
sum = h->key_sum (h, key);
break;
e = string_key_equal (h, key1, key2);
break;
+ case KEY_FUNC_MEM:
+ e = mem_key_equal (h, key1, key2);
+ break;
+
default:
e = h->key_equal (h, key1, key2);
break;
log2_bytes = 1 + hash_pair_log2_bytes (h);
q = clib_mem_alloc (1ULL << log2_bytes);
}
- clib_memcpy (q, &p->direct, hash_pair_bytes (h));
+ clib_memcpy_fast (q, &p->direct, hash_pair_bytes (h));
pi->pairs = q;
if (h->log2_pair_size > 0)
if (len == 2)
{
- clib_memcpy (p, q == r ? hash_forward1 (h, r) : r,
- hash_pair_bytes (h));
+ clib_memcpy_fast (p, q == r ? hash_forward1 (h, r) : r,
+ hash_pair_bytes (h));
set_is_user (v, i, 1);
}
else
{
/* If deleting a pair we need to keep non-null pairs together. */
if (q < e)
- clib_memcpy (q, e, hash_pair_bytes (h));
+ clib_memcpy_fast (q, e, hash_pair_bytes (h));
else
zero_pair (h, q);
if (is_vec)
hash_t *h = hash_header (v);
hash_pair_union_t *p = 0;
uword found_key = 0;
+ uword value_bytes;
uword i;
if (!v)
i = key_sum (h, key) & (_vec_len (v) - 1);
p = get_pair (v, i);
+ value_bytes = hash_value_bytes (h);
if (hash_is_user (v, i))
{
if (op == UNSET)
{
set_is_user (v, i, 0);
- if (old_value)
- clib_memcpy (old_value, p->direct.value,
- hash_value_bytes (h));
+ if (old_value && value_bytes)
+ clib_memcpy_fast (old_value, p->direct.value, value_bytes);
zero_pair (h, &p->direct);
}
}
found_key = p != 0;
if (found_key && op == UNSET)
{
- if (old_value)
- clib_memcpy (old_value, &p->direct.value,
- hash_value_bytes (h));
+ if (old_value && value_bytes)
+ clib_memcpy_fast (old_value, &p->direct.value, value_bytes);
unset_indirect (v, i, &p->direct);
}
}
- if (op == SET && p != 0)
+ if (op == SET && p != 0 && value_bytes)
{
/* Save away old value for caller. */
if (old_value && found_key)
- clib_memcpy (old_value, &p->direct.value, hash_value_bytes (h));
- clib_memcpy (&p->direct.value, new_value, hash_value_bytes (h));
+ clib_memcpy_fast (old_value, &p->direct.value, value_bytes);
+ clib_memcpy_fast (&p->direct.value, new_value, value_bytes);
}
if (op == SET)
}
/* Fetch value of key. */
-uword *
+__clib_export uword *
_hash_get (void *v, uword key)
{
hash_t *h = hash_header (v);
return &p->value[0];
}
-hash_pair_t *
+__clib_export hash_pair_t *
_hash_get_pair (void *v, uword key)
{
return lookup (v, key, GET, 0, 0);
}
-hash_pair_t *
-hash_next (void *v, hash_next_t * hn)
+__clib_export hash_pair_t *
+hash_next (void *v, hash_next_t *hn)
{
hash_t *h = hash_header (v);
hash_pair_t *p;
{
/* Restore flags. */
h->flags = hn->f;
- memset (hn, 0, sizeof (hn[0]));
+ clib_memset (hn, 0, sizeof (hn[0]));
return 0;
}
}
/* Remove key from table. */
-void *
+__clib_export void *
_hash_unset (void *v, uword key, void *old_value)
{
hash_t *h;
return v;
}
-void *
+__clib_export void *
_hash_create (uword elts, hash_t * h_user)
{
hash_t *h;
if (h_user)
log2_pair_size = h_user->log2_pair_size;
- v = _vec_resize (0,
+ v = _vec_resize ((void *) 0,
/* vec len: */ elts,
/* data bytes: */
(elts << log2_pair_size) * sizeof (hash_pair_t),
return v;
}
-void *
+__clib_export void *
_hash_free (void *v)
{
hash_t *h = hash_header (v);
return new;
}
-void *
+__clib_export void *
hash_resize (void *old, uword new_size)
{
return hash_resize_internal (old, new_size, 1);
}
-void *
+__clib_export void *
hash_dup (void *old)
{
return hash_resize_internal (old, vec_len (old), 0);
}
-void *
+__clib_export void *
_hash_set3 (void *v, uword key, void *value, void *old_value)
{
hash_t *h;
return v;
}
-uword
+__clib_export uword
vec_key_sum (hash_t * h, uword key)
{
void *v = uword_to_pointer (key, void *);
return hash_memory (v, vec_len (v) * h->user, 0);
}
-uword
+__clib_export uword
vec_key_equal (hash_t * h, uword key1, uword key2)
{
void *v1 = uword_to_pointer (key1, void *);
return l1 == l2 && 0 == memcmp (v1, v2, l1 * h->user);
}
-u8 *
+__clib_export u8 *
vec_key_format_pair (u8 * s, va_list * args)
{
void *CLIB_UNUSED (user_arg) = va_arg (*args, void *);
return s;
}
-uword
+__clib_export uword
mem_key_sum (hash_t * h, uword key)
{
uword *v = uword_to_pointer (key, void *);
return hash_memory (v, h->user, 0);
}
-uword
+__clib_export uword
mem_key_equal (hash_t * h, uword key1, uword key2)
{
void *v1 = uword_to_pointer (key1, void *);
return s;
}
-uword
+__clib_export uword
hash_bytes (void *v)
{
uword i, bytes;
return bytes;
}
-u8 *
-format_hash (u8 * s, va_list * va)
+__clib_export u8 *
+format_hash (u8 *s, va_list *va)
{
void *v = va_arg (*va, void *);
int verbose = va_arg (*va, int);
return p ? 1 : 0;
}
-uword
+__clib_export uword
unformat_hash_vec_string (unformat_input_t * input, va_list * va)
{
return unformat_hash_string_internal (input, va, /* is_vec */ 1);
}
-uword
+__clib_export uword
unformat_hash_string (unformat_input_t * input, va_list * va)
{
return unformat_hash_string_internal (input, va, /* is_vec */ 0);
}
-clib_error_t *
+__clib_export clib_error_t *
hash_validate (void *v)
{
hash_t *h = hash_header (v);
Code Review / vpp.git / blobdiff