2 Copyright (c) 2013 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #ifndef included_clib_pfhash_h
18 #define included_clib_pfhash_h
21 #include <vppinfra/clib.h>
22 #include <vppinfra/hash.h>
23 #include <vppinfra/pool.h>
25 #if defined(CLIB_HAVE_VEC128) && ! defined (__ALTIVEC__)
28 /* 3 x 16 = 48 key bytes */
33 /* 3 x 4 = 12 value bytes */
39 /* 5 x 8 = 40 key bytes */
44 /* 5 x 4 = 20 value bytes */
50 /* 4 x 8 = 32 key bytes */
55 /* 4 x 8 = 32 value bytes */
60 /* 8 x 4 = 32 key bytes */
66 /* 8 x 4 = 32 value bytes */
73 pfhash_kv_8v8_t kv8v8;
80 #define PFHASH_BUCKET_OVERFLOW (u32)~0
82 /* Pool of key/value pairs */
85 /* overflow plain-o-hash */
86 uword * overflow_hash;
88 /* Pretty-print name */
96 u32 nitems_in_overflow;
99 void pfhash_init (pfhash_t * p, char * name, u32 key_size, u32 value_size,
101 void pfhash_free (pfhash_t * p);
102 u64 pfhash_get (pfhash_t * p, u32 bucket, void * key);
103 void pfhash_set (pfhash_t * p, u32 bucket, void * key, void * value);
104 void pfhash_unset (pfhash_t * p, u32 bucket, void * key);
106 format_function_t format_pfhash;
108 static inline void pfhash_prefetch_bucket (pfhash_t * p, u32 bucket)
109 { CLIB_PREFETCH (&p->buckets[bucket], CLIB_CACHE_LINE_BYTES, LOAD); }
111 static inline u32 pfhash_read_bucket_prefetch_kv (pfhash_t * p, u32 bucket)
113 u32 bucket_contents = p->buckets[bucket];
114 if (PREDICT_TRUE ((bucket_contents & PFHASH_BUCKET_OVERFLOW) == 0))
115 CLIB_PREFETCH (&p->kvp[bucket_contents], CLIB_CACHE_LINE_BYTES, LOAD);
116 return bucket_contents;
120 * pfhash_search_kv_16
121 * See if the supplied 16-byte key matches one of three 16-byte (key,value) pairs.
122 * Return the indicated value, or ~0 if no match
124 * Note: including the overflow test, the fast path is 35 instrs
125 * on x86_64. Elves will steal your keyboard in the middle of the night if
126 * you "improve" it without checking the generated code!
128 static inline u32 pfhash_search_kv_16 (pfhash_t * p, u32 bucket_contents,
131 u32x4 diff0, diff1, diff2;
132 u32 is_equal0, is_equal1, is_equal2;
137 if (PREDICT_FALSE (bucket_contents == PFHASH_BUCKET_OVERFLOW))
140 hp = hash_get_mem (p->overflow_hash, key);
146 kv = &p->kvp[bucket_contents].kv16;
148 diff0 = u32x4_sub (kv->kb.k_u32x4[0], key[0]);
149 diff1 = u32x4_sub (kv->kb.k_u32x4[1], key[0]);
150 diff2 = u32x4_sub (kv->kb.k_u32x4[2], key[0]);
152 no_match = is_equal0 = (i16) u32x4_zero_byte_mask (diff0);
153 is_equal1 = (i16) u32x4_zero_byte_mask (diff1);
154 no_match |= is_equal1;
155 is_equal2 = (i16) u32x4_zero_byte_mask (diff2);
156 no_match |= is_equal2;
157 /* If any of the three items matched, no_match will be zero after this line */
158 no_match = ~no_match;
160 rv = (is_equal0 & kv->values[0])
161 |(is_equal1 & kv->values[1])
162 | (is_equal2 & kv->values[2])
168 static inline u32 pfhash_search_kv_8 (pfhash_t * p, u32 bucket_contents,
174 if (PREDICT_FALSE (bucket_contents == PFHASH_BUCKET_OVERFLOW))
177 hp = hash_get_mem (p->overflow_hash, key);
183 kv = &p->kvp[bucket_contents].kv8;
185 rv = (kv->kb.k_u64[0] == key[0]) ? kv->values[0] : rv;
186 rv = (kv->kb.k_u64[1] == key[0]) ? kv->values[1] : rv;
187 rv = (kv->kb.k_u64[2] == key[0]) ? kv->values[2] : rv;
188 rv = (kv->kb.k_u64[3] == key[0]) ? kv->values[3] : rv;
189 rv = (kv->kb.k_u64[4] == key[0]) ? kv->values[4] : rv;
194 static inline u64 pfhash_search_kv_8v8 (pfhash_t * p, u32 bucket_contents,
200 if (PREDICT_FALSE (bucket_contents == PFHASH_BUCKET_OVERFLOW))
203 hp = hash_get_mem (p->overflow_hash, key);
209 kv = &p->kvp[bucket_contents].kv8v8;
211 rv = (kv->kb.k_u64[0] == key[0]) ? kv->values[0] : rv;
212 rv = (kv->kb.k_u64[1] == key[0]) ? kv->values[1] : rv;
213 rv = (kv->kb.k_u64[2] == key[0]) ? kv->values[2] : rv;
214 rv = (kv->kb.k_u64[3] == key[0]) ? kv->values[3] : rv;
219 static inline u32 pfhash_search_kv_4 (pfhash_t * p, u32 bucket_contents,
224 u32 zbm[2], winner_index;
227 if (PREDICT_FALSE (bucket_contents == PFHASH_BUCKET_OVERFLOW))
230 hp = hash_get_mem (p->overflow_hash, key);
236 kv = &p->kvp[bucket_contents].kv4;
238 vector_key = u32x4_splat (key[0]);
240 is_equal[0] = u32x4_is_equal (kv->kb.k_u32x4[0], vector_key);
241 is_equal[1] = u32x4_is_equal (kv->kb.k_u32x4[1], vector_key);
242 zbm[0] = ~u32x4_zero_byte_mask (is_equal[0]) & 0xFFFF;
243 zbm[1] = ~u32x4_zero_byte_mask (is_equal[1]) & 0xFFFF;
245 if (PREDICT_FALSE((zbm[0] == 0) && (zbm[1] == 0)))
248 winner_index = min_log2 (zbm[0])>>2;
249 winner_index = zbm[1] ? (4 + (min_log2 (zbm[1])>>2)) : winner_index;
251 return kv->values[winner_index];
254 #endif /* CLIB_HAVE_VEC128 */
256 #endif /* included_clib_pfhash_h */