X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvppinfra%2Fhash.c;h=220c16989de3a37731145040554fa45eac66ac31;hb=97d6860b229e15c5d34931f6ca440adb32e600dc;hp=79103b6d3f48eb0826bff64d782e164600e69040;hpb=2f25ef33c870869e613b81ee7603b9b7337e48fe;p=vpp.git

diff --git a/src/vppinfra/hash.c b/src/vppinfra/hash.c
index 79103b6d3f4..220c16989de 100644
--- a/src/vppinfra/hash.c
+++ b/src/vppinfra/hash.c
@@ -43,13 +43,13 @@
 always_inline void
 zero_pair (hash_t * h, hash_pair_t * p)
 {
-  memset (p, 0, hash_pair_bytes (h));
+  clib_memset (p, 0, hash_pair_bytes (h));
 }
 
 always_inline void
 init_pair (hash_t * h, hash_pair_t * p)
 {
-  memset (p->value, ~0, hash_value_bytes (h));
+  clib_memset (p->value, ~0, hash_value_bytes (h));
 }
 
 always_inline hash_pair_union_t *
@@ -103,14 +103,32 @@ zap64 (u64 x, word n)
  * Therefore all the 8 Bytes of the u64 are systematically read, which
  * rightfully causes address-sanitizer to raise an error on smaller inputs.
  *
- * However the invalid Bytes are discarded within zap64(), whicj is why
+ * However the invalid Bytes are discarded within zap64(), which is why
  * this can be silenced safely.
+ *
+ * The above is true *unless* the extra bytes cross a page boundary
+ * into unmapped or no-access space, hence the boundary crossing check.
  */
-static inline u64 __attribute__ ((no_sanitize_address))
+static inline u64
 hash_memory64 (void *p, word n_bytes, u64 state)
 {
   u64 *q = p;
   u64 a, b, c, n;
+  int page_boundary_crossing;
+  u64 start_addr, end_addr;
+  union
+  {
+    u8 as_u8[8];
+    u64 as_u64;
+  } tmp;
+
+  /*
+   * If the request crosses a 4k boundary, it's not OK to assume
+   * that the zap64 game is safe. 4k is the minimum known page size.
+   */
+  start_addr = (u64) p;
+  end_addr = start_addr + n_bytes + 7;
+  page_boundary_crossing = (start_addr >> 12) != (end_addr >> 12);
 
   a = b = 0x9e3779b97f4a7c13LL;
   c = state;
@@ -133,18 +151,51 @@ hash_memory64 (void *p, word n_bytes, u64 state)
       a += clib_mem_unaligned (q + 0, u64);
       b += clib_mem_unaligned (q + 1, u64);
       if (n % sizeof (u64))
-	c += zap64 (clib_mem_unaligned (q + 2, u64), n % sizeof (u64)) << 8;
+	{
+	  if (PREDICT_TRUE (page_boundary_crossing == 0))
+	    c +=
+	      zap64 (CLIB_MEM_OVERFLOW
+		     (clib_mem_unaligned (q + 2, u64), q + 2, sizeof (u64)),
+		     n % sizeof (u64)) << 8;
+	  else
+	    {
+	      clib_memcpy_fast (tmp.as_u8, q + 2, n % sizeof (u64));
+	      c += zap64 (tmp.as_u64, n % sizeof (u64)) << 8;
+	    }
+	}
       break;
 
     case 1:
       a += clib_mem_unaligned (q + 0, u64);
       if (n % sizeof (u64))
-	b += zap64 (clib_mem_unaligned (q + 1, u64), n % sizeof (u64));
+	{
+	  if (PREDICT_TRUE (page_boundary_crossing == 0))
+	    b +=
+	      zap64 (CLIB_MEM_OVERFLOW
+		     (clib_mem_unaligned (q + 1, u64), q + 1, sizeof (u64)),
+		     n % sizeof (u64));
+	  else
+	    {
+	      clib_memcpy_fast (tmp.as_u8, q + 1, n % sizeof (u64));
+	      b += zap64 (tmp.as_u64, n % sizeof (u64));
+	    }
+	}
       break;
 
     case 0:
       if (n % sizeof (u64))
-	a += zap64 (clib_mem_unaligned (q + 0, u64), n % sizeof (u64));
+	{
+	  if (PREDICT_TRUE (page_boundary_crossing == 0))
+	    a +=
+	      zap64 (CLIB_MEM_OVERFLOW
+		     (clib_mem_unaligned (q + 0, u64), q + 0, sizeof (u64)),
+		     n % sizeof (u64));
+	  else
+	    {
+	      clib_memcpy_fast (tmp.as_u8, q, n % sizeof (u64));
+	      a += zap64 (tmp.as_u64, n % sizeof (u64));
+	    }
+	}
       break;
     }
 
@@ -220,7 +271,7 @@ hash_memory32 (void *p, word n_bytes, u32 state)
 }
 #endif
 
-uword
+__clib_export uword
 hash_memory (void *p, word n_bytes, uword state)
 {
   uword *q = p;
@@ -282,6 +333,10 @@ key_sum (hash_t * h, uword key)
       sum = string_key_sum (h, key);
       break;
 
+    case KEY_FUNC_MEM:
+      sum = mem_key_sum (h, key);
+      break;
+
     default:
       sum = h->key_sum (h, key);
       break;
@@ -312,6 +367,10 @@ key_equal1 (hash_t * h, uword key1, uword key2, uword e)
       e = string_key_equal (h, key1, key2);
       break;
 
+    case KEY_FUNC_MEM:
+      e = mem_key_equal (h, key1, key2);
+      break;
+
     default:
       e = h->key_equal (h, key1, key2);
       break;
@@ -368,7 +427,7 @@ set_indirect_is_user (void *v, uword i, hash_pair_union_t * p, uword key)
       log2_bytes = 1 + hash_pair_log2_bytes (h);
       q = clib_mem_alloc (1ULL << log2_bytes);
     }
-  clib_memcpy (q, &p->direct, hash_pair_bytes (h));
+  clib_memcpy_fast (q, &p->direct, hash_pair_bytes (h));
 
   pi->pairs = q;
   if (h->log2_pair_size > 0)
@@ -449,8 +508,8 @@ unset_indirect (void *v, uword i, hash_pair_t * q)
 
       if (len == 2)
 	{
-	  clib_memcpy (p, q == r ? hash_forward1 (h, r) : r,
-		       hash_pair_bytes (h));
+	  clib_memcpy_fast (p, q == r ? hash_forward1 (h, r) : r,
+			    hash_pair_bytes (h));
 	  set_is_user (v, i, 1);
 	}
       else
@@ -465,7 +524,7 @@ unset_indirect (void *v, uword i, hash_pair_t * q)
     {
       /* If deleting a pair we need to keep non-null pairs together. */
       if (q < e)
-	clib_memcpy (q, e, hash_pair_bytes (h));
+	clib_memcpy_fast (q, e, hash_pair_bytes (h));
       else
 	zero_pair (h, q);
       if (is_vec)
@@ -506,8 +565,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
 	    {
 	      set_is_user (v, i, 0);
 	      if (old_value)
-		clib_memcpy (old_value, p->direct.value,
-			     hash_value_bytes (h));
+		clib_memcpy_fast (old_value, p->direct.value,
+				  hash_value_bytes (h));
 	      zero_pair (h, &p->direct);
 	    }
 	}
@@ -540,8 +599,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
 	  if (found_key && op == UNSET)
 	    {
 	      if (old_value)
-		clib_memcpy (old_value, &p->direct.value,
-			     hash_value_bytes (h));
+		clib_memcpy_fast (old_value, &p->direct.value,
+				  hash_value_bytes (h));
 
 	      unset_indirect (v, i, &p->direct);
 
@@ -556,8 +615,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
     {
       /* Save away old value for caller. */
       if (old_value && found_key)
-	clib_memcpy (old_value, &p->direct.value, hash_value_bytes (h));
-      clib_memcpy (&p->direct.value, new_value, hash_value_bytes (h));
+	clib_memcpy_fast (old_value, &p->direct.value, hash_value_bytes (h));
+      clib_memcpy_fast (&p->direct.value, new_value, hash_value_bytes (h));
     }
 
   if (op == SET)
@@ -569,7 +628,7 @@ lookup (void *v, uword key, enum lookup_opcode op,
 }
 
 /* Fetch value of key. */
-uword *
+__clib_export uword *
 _hash_get (void *v, uword key)
 {
   hash_t *h = hash_header (v);
@@ -588,7 +647,7 @@ _hash_get (void *v, uword key)
     return &p->value[0];
 }
 
-hash_pair_t *
+__clib_export hash_pair_t *
 _hash_get_pair (void *v, uword key)
 {
   return lookup (v, key, GET, 0, 0);
@@ -616,7 +675,7 @@ hash_next (void *v, hash_next_t * hn)
 	{
 	  /* Restore flags. */
 	  h->flags = hn->f;
-	  memset (hn, 0, sizeof (hn[0]));
+	  clib_memset (hn, 0, sizeof (hn[0]));
 	  return 0;
 	}
 
@@ -648,7 +707,7 @@ hash_next (void *v, hash_next_t * hn)
 }
 
 /* Remove key from table. */
-void *
+__clib_export void *
 _hash_unset (void *v, uword key, void *old_value)
 {
   hash_t *h;
@@ -669,7 +728,7 @@ _hash_unset (void *v, uword key, void *old_value)
   return v;
 }
 
-void *
+__clib_export void *
 _hash_create (uword elts, hash_t * h_user)
 {
   hash_t *h;
@@ -715,7 +774,7 @@ _hash_create (uword elts, hash_t * h_user)
   return v;
 }
 
-void *
+__clib_export void *
 _hash_free (void *v)
 {
   hash_t *h = hash_header (v);
@@ -771,13 +830,13 @@ hash_resize (void *old, uword new_size)
   return hash_resize_internal (old, new_size, 1);
 }
 
-void *
+__clib_export void *
 hash_dup (void *old)
 {
   return hash_resize_internal (old, vec_len (old), 0);
 }
 
-void *
+__clib_export void *
 _hash_set3 (void *v, uword key, void *value, void *old_value)
 {
   hash_t *h;
@@ -798,14 +857,14 @@ _hash_set3 (void *v, uword key, void *value, void *old_value)
   return v;
 }
 
-uword
+__clib_export uword
 vec_key_sum (hash_t * h, uword key)
 {
   void *v = uword_to_pointer (key, void *);
   return hash_memory (v, vec_len (v) * h->user, 0);
 }
 
-uword
+__clib_export uword
 vec_key_equal (hash_t * h, uword key1, uword key2)
 {
   void *v1 = uword_to_pointer (key1, void *);
@@ -815,7 +874,7 @@ vec_key_equal (hash_t * h, uword key1, uword key2)
   return l1 == l2 && 0 == memcmp (v1, v2, l1 * h->user);
 }
 
-u8 *
+__clib_export u8 *
 vec_key_format_pair (u8 * s, va_list * args)
 {
   void *CLIB_UNUSED (user_arg) = va_arg (*args, void *);
@@ -866,14 +925,14 @@ vec_key_format_pair (u8 * s, va_list * args)
   return s;
 }
 
-uword
+__clib_export uword
 mem_key_sum (hash_t * h, uword key)
 {
   uword *v = uword_to_pointer (key, void *);
   return hash_memory (v, h->user, 0);
 }
 
-uword
+__clib_export uword
 mem_key_equal (hash_t * h, uword key1, uword key2)
 {
   void *v1 = uword_to_pointer (key1, void *);
@@ -931,7 +990,7 @@ hash_format_pair_default (u8 * s, va_list * args)
   return s;
 }
 
-uword
+__clib_export uword
 hash_bytes (void *v)
 {
   uword i, bytes;
@@ -1039,13 +1098,13 @@ unformat_hash_string_internal (unformat_input_t * input,
   return p ? 1 : 0;
 }
 
-uword
+__clib_export uword
 unformat_hash_vec_string (unformat_input_t * input, va_list * va)
 {
   return unformat_hash_string_internal (input, va, /* is_vec */ 1);
 }
 
-uword
+__clib_export uword
 unformat_hash_string (unformat_input_t * input, va_list * va)
 {
   return unformat_hash_string_internal (input, va, /* is_vec */ 0);