X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fhash.c;h=6115b0cffd6cb575c672f754d74d1b9de4ecb03f;hb=dbf68c9aa258238260df34c0e864223ea4f3a987;hp=121fa38570553c3f72d603563b2c80c3b73c63e6;hpb=c5c2bb3f042fe11184faa731b6209444e2c05231;p=vpp.git diff --git a/src/vppinfra/hash.c b/src/vppinfra/hash.c index 121fa385705..6115b0cffd6 100644 --- a/src/vppinfra/hash.c +++ b/src/vppinfra/hash.c @@ -43,13 +43,13 @@ always_inline void zero_pair (hash_t * h, hash_pair_t * p) { - memset (p, 0, hash_pair_bytes (h)); + clib_memset (p, 0, hash_pair_bytes (h)); } always_inline void init_pair (hash_t * h, hash_pair_t * p) { - memset (p->value, ~0, hash_value_bytes (h)); + clib_memset (p->value, ~0, hash_value_bytes (h)); } always_inline hash_pair_union_t * @@ -103,14 +103,32 @@ zap64 (u64 x, word n) * Therefore all the 8 Bytes of the u64 are systematically read, which * rightfully causes address-sanitizer to raise an error on smaller inputs. * - * However the invalid Bytes are discarded within zap64(), whicj is why + * However the invalid Bytes are discarded within zap64(), which is why * this can be silenced safely. + * + * The above is true *unless* the extra bytes cross a page boundary + * into unmapped or no-access space, hence the boundary crossing check. */ -static inline u64 __attribute__ ((no_sanitize_address)) +static inline u64 hash_memory64 (void *p, word n_bytes, u64 state) { u64 *q = p; u64 a, b, c, n; + int page_boundary_crossing; + u64 start_addr, end_addr; + union + { + u8 as_u8[8]; + u64 as_u64; + } tmp; + + /* + * If the request crosses a 4k boundary, it's not OK to assume + * that the zap64 game is safe. 4k is the minimum known page size. + */ + start_addr = (u64) p; + end_addr = start_addr + n_bytes + 7; + page_boundary_crossing = (start_addr >> 12) != (end_addr >> 12); a = b = 0x9e3779b97f4a7c13LL; c = state; @@ -133,18 +151,51 @@ hash_memory64 (void *p, word n_bytes, u64 state) a += clib_mem_unaligned (q + 0, u64); b += clib_mem_unaligned (q + 1, u64); if (n % sizeof (u64)) - c += zap64 (clib_mem_unaligned (q + 2, u64), n % sizeof (u64)) << 8; + { + if (PREDICT_TRUE (page_boundary_crossing == 0)) + c += + zap64 (CLIB_MEM_OVERFLOW + (clib_mem_unaligned (q + 2, u64), q + 2, sizeof (u64)), + n % sizeof (u64)) << 8; + else + { + clib_memcpy_fast (tmp.as_u8, q + 2, n % sizeof (u64)); + c += zap64 (tmp.as_u64, n % sizeof (u64)) << 8; + } + } break; case 1: a += clib_mem_unaligned (q + 0, u64); if (n % sizeof (u64)) - b += zap64 (clib_mem_unaligned (q + 1, u64), n % sizeof (u64)); + { + if (PREDICT_TRUE (page_boundary_crossing == 0)) + b += + zap64 (CLIB_MEM_OVERFLOW + (clib_mem_unaligned (q + 1, u64), q + 1, sizeof (u64)), + n % sizeof (u64)); + else + { + clib_memcpy_fast (tmp.as_u8, q + 1, n % sizeof (u64)); + b += zap64 (tmp.as_u64, n % sizeof (u64)); + } + } break; case 0: if (n % sizeof (u64)) - a += zap64 (clib_mem_unaligned (q + 0, u64), n % sizeof (u64)); + { + if (PREDICT_TRUE (page_boundary_crossing == 0)) + a += + zap64 (CLIB_MEM_OVERFLOW + (clib_mem_unaligned (q + 0, u64), q + 0, sizeof (u64)), + n % sizeof (u64)); + else + { + clib_memcpy_fast (tmp.as_u8, q, n % sizeof (u64)); + a += zap64 (tmp.as_u64, n % sizeof (u64)); + } + } break; } @@ -282,6 +333,10 @@ key_sum (hash_t * h, uword key) sum = string_key_sum (h, key); break; + case KEY_FUNC_MEM: + sum = mem_key_sum (h, key); + break; + default: sum = h->key_sum (h, key); break; @@ -312,6 +367,10 @@ key_equal1 (hash_t * h, uword key1, uword key2, uword e) e = string_key_equal (h, key1, key2); break; + case KEY_FUNC_MEM: + e = mem_key_equal (h, key1, key2); + break; + default: e = h->key_equal (h, key1, key2); break; @@ -368,7 +427,7 @@ set_indirect_is_user (void *v, uword i, hash_pair_union_t * p, uword key) log2_bytes = 1 + hash_pair_log2_bytes (h); q = clib_mem_alloc (1ULL << log2_bytes); } - clib_memcpy (q, &p->direct, hash_pair_bytes (h)); + clib_memcpy_fast (q, &p->direct, hash_pair_bytes (h)); pi->pairs = q; if (h->log2_pair_size > 0) @@ -449,8 +508,8 @@ unset_indirect (void *v, uword i, hash_pair_t * q) if (len == 2) { - clib_memcpy (p, q == r ? hash_forward1 (h, r) : r, - hash_pair_bytes (h)); + clib_memcpy_fast (p, q == r ? hash_forward1 (h, r) : r, + hash_pair_bytes (h)); set_is_user (v, i, 1); } else @@ -465,7 +524,7 @@ unset_indirect (void *v, uword i, hash_pair_t * q) { /* If deleting a pair we need to keep non-null pairs together. */ if (q < e) - clib_memcpy (q, e, hash_pair_bytes (h)); + clib_memcpy_fast (q, e, hash_pair_bytes (h)); else zero_pair (h, q); if (is_vec) @@ -506,8 +565,8 @@ lookup (void *v, uword key, enum lookup_opcode op, { set_is_user (v, i, 0); if (old_value) - clib_memcpy (old_value, p->direct.value, - hash_value_bytes (h)); + clib_memcpy_fast (old_value, p->direct.value, + hash_value_bytes (h)); zero_pair (h, &p->direct); } } @@ -540,8 +599,8 @@ lookup (void *v, uword key, enum lookup_opcode op, if (found_key && op == UNSET) { if (old_value) - clib_memcpy (old_value, &p->direct.value, - hash_value_bytes (h)); + clib_memcpy_fast (old_value, &p->direct.value, + hash_value_bytes (h)); unset_indirect (v, i, &p->direct); @@ -556,8 +615,8 @@ lookup (void *v, uword key, enum lookup_opcode op, { /* Save away old value for caller. */ if (old_value && found_key) - clib_memcpy (old_value, &p->direct.value, hash_value_bytes (h)); - clib_memcpy (&p->direct.value, new_value, hash_value_bytes (h)); + clib_memcpy_fast (old_value, &p->direct.value, hash_value_bytes (h)); + clib_memcpy_fast (&p->direct.value, new_value, hash_value_bytes (h)); } if (op == SET) @@ -616,7 +675,7 @@ hash_next (void *v, hash_next_t * hn) { /* Restore flags. */ h->flags = hn->f; - memset (hn, 0, sizeof (hn[0])); + clib_memset (hn, 0, sizeof (hn[0])); return 0; } @@ -685,7 +744,7 @@ _hash_create (uword elts, hash_t * h_user) if (h_user) log2_pair_size = h_user->log2_pair_size; - v = _vec_resize (0, + v = _vec_resize ((void *) 0, /* vec len: */ elts, /* data bytes: */ (elts << log2_pair_size) * sizeof (hash_pair_t),