2 *------------------------------------------------------------------
3 * Copyright (c) 2019 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <crypto_native/crypto_native.h>
22 #include <crypto_native/aes.h>
23 #include <crypto_native/ghash.h>
25 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
26 #pragma GCC optimize ("O3")
31 /* pre-calculated hash key values */
33 /* extracted AES key */
37 static_always_inline void
38 aesni_gcm_load (u8x16 * d, u8x16u * inv, int n, int n_bytes)
40 for (int i = 0; i < n - 1; i++)
42 d[n - 1] = n_bytes ? aes_load_partial (inv + n - 1, n_bytes) : inv[n - 1];
45 static_always_inline void
46 aesni_gcm_store (u8x16 * d, u8x16u * outv, int n, int n_bytes)
48 for (int i = 0; i < n - 1; i++)
51 aes_store_partial (outv + n - 1, d[n - 1], n_bytes);
53 outv[n - 1] = d[n - 1];
56 static_always_inline void
57 aesni_gcm_enc_first_round (u8x16 * r, u32x4 * Y, u32 * ctr, u8x16 k,
60 static const u32x4 last_byte_one = { 0, 0, 0, 1 << 24 };
62 if (PREDICT_TRUE ((u8) ctr[0] < (256 - n_blocks)))
64 for (int i = 0; i < n_blocks; i++)
66 Y[0] += last_byte_one;
67 r[i] = k ^ (u8x16) Y[0];
73 for (int i = 0; i < n_blocks; i++)
75 Y[0][3] = clib_host_to_net_u32 (++ctr[0]);
76 r[i] = k ^ (u8x16) Y[0];
81 static_always_inline void
82 aesni_gcm_enc_round (u8x16 * r, u8x16 k, int n_blocks)
84 for (int i = 0; i < n_blocks; i++)
85 r[i] = aes_enc_round (r[i], k);
88 static_always_inline void
89 aesni_gcm_enc_last_round (u8x16 * r, u8x16 * d, u8x16 const *k,
90 int rounds, int n_blocks)
93 /* additional ronuds for AES-192 and AES-256 */
94 for (int i = 10; i < rounds; i++)
95 aesni_gcm_enc_round (r, k[i], n_blocks);
97 for (int i = 0; i < n_blocks; i++)
98 d[i] ^= aes_enc_last_round (r[i], k[rounds]);
101 static_always_inline u8x16
102 aesni_gcm_ghash_blocks (u8x16 T, aes_gcm_key_data_t * kd,
103 u8x16u * in, int n_blocks)
105 ghash_data_t _gd, *gd = &_gd;
106 const u8x16 *Hi = kd->Hi + n_blocks - 1;
107 ghash_mul_first (gd, u8x16_reflect (in[0]) ^ T, Hi[0]);
108 for (int i = 1; i < n_blocks; i++)
109 ghash_mul_next (gd, u8x16_reflect ((in[i])), Hi[-i]);
112 return ghash_final (gd);
115 static_always_inline u8x16
116 aesni_gcm_ghash (u8x16 T, aes_gcm_key_data_t * kd, u8x16u * in, u32 n_left)
119 while (n_left >= 128)
121 T = aesni_gcm_ghash_blocks (T, kd, in, 8);
128 T = aesni_gcm_ghash_blocks (T, kd, in, 4);
135 T = aesni_gcm_ghash_blocks (T, kd, in, 2);
142 T = aesni_gcm_ghash_blocks (T, kd, in, 1);
149 u8x16 r = aes_load_partial (in, n_left);
150 T = ghash_mul (u8x16_reflect (r) ^ T, kd->Hi[0]);
155 static_always_inline u8x16
156 aesni_gcm_calc (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
157 u32x4 * Y, u32 * ctr, u8x16u * inv, u8x16u * outv,
158 int rounds, int n, int last_block_bytes, int with_ghash,
162 ghash_data_t _gd = { }, *gd = &_gd;
163 const u8x16 *rk = (u8x16 *) kd->Ke;
164 int hidx = is_encrypt ? 4 : n, didx = 0;
166 clib_prefetch_load (inv + 4);
168 /* AES rounds 0 and 1 */
169 aesni_gcm_enc_first_round (r, Y, ctr, rk[0], n);
170 aesni_gcm_enc_round (r, rk[1], n);
172 /* load data - decrypt round */
174 aesni_gcm_load (d, inv, n, last_block_bytes);
176 /* GHASH multiply block 1 */
178 ghash_mul_first (gd, u8x16_reflect (d[didx++]) ^ T, kd->Hi[--hidx]);
180 /* AES rounds 2 and 3 */
181 aesni_gcm_enc_round (r, rk[2], n);
182 aesni_gcm_enc_round (r, rk[3], n);
184 /* GHASH multiply block 2 */
185 if (with_ghash && hidx)
186 ghash_mul_next (gd, u8x16_reflect (d[didx++]), kd->Hi[--hidx]);
188 /* AES rounds 4 and 5 */
189 aesni_gcm_enc_round (r, rk[4], n);
190 aesni_gcm_enc_round (r, rk[5], n);
192 /* GHASH multiply block 3 */
193 if (with_ghash && hidx)
194 ghash_mul_next (gd, u8x16_reflect (d[didx++]), kd->Hi[--hidx]);
196 /* AES rounds 6 and 7 */
197 aesni_gcm_enc_round (r, rk[6], n);
198 aesni_gcm_enc_round (r, rk[7], n);
200 /* GHASH multiply block 4 */
201 if (with_ghash && hidx)
202 ghash_mul_next (gd, u8x16_reflect (d[didx++]), kd->Hi[--hidx]);
204 /* AES rounds 8 and 9 */
205 aesni_gcm_enc_round (r, rk[8], n);
206 aesni_gcm_enc_round (r, rk[9], n);
208 /* GHASH reduce 1st step */
212 /* load data - encrypt round */
214 aesni_gcm_load (d, inv, n, last_block_bytes);
216 /* GHASH reduce 2nd step */
220 /* AES last round(s) */
221 aesni_gcm_enc_last_round (r, d, rk, rounds, n);
224 aesni_gcm_store (d, outv, n, last_block_bytes);
226 /* GHASH final step */
228 T = ghash_final (gd);
233 static_always_inline u8x16
234 aesni_gcm_calc_double (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
235 u32x4 * Y, u32 * ctr, u8x16u * inv, u8x16u * outv,
236 int rounds, int is_encrypt)
239 ghash_data_t _gd, *gd = &_gd;
240 const u8x16 *rk = (u8x16 *) kd->Ke;
242 /* AES rounds 0 and 1 */
243 aesni_gcm_enc_first_round (r, Y, ctr, rk[0], 4);
244 aesni_gcm_enc_round (r, rk[1], 4);
246 /* load 4 blocks of data - decrypt round */
248 aesni_gcm_load (d, inv, 4, 0);
250 /* GHASH multiply block 0 */
251 ghash_mul_first (gd, u8x16_reflect (d[0]) ^ T, kd->Hi[7]);
253 /* AES rounds 2 and 3 */
254 aesni_gcm_enc_round (r, rk[2], 4);
255 aesni_gcm_enc_round (r, rk[3], 4);
257 /* GHASH multiply block 1 */
258 ghash_mul_next (gd, u8x16_reflect (d[1]), kd->Hi[6]);
260 /* AES rounds 4 and 5 */
261 aesni_gcm_enc_round (r, rk[4], 4);
262 aesni_gcm_enc_round (r, rk[5], 4);
264 /* GHASH multiply block 2 */
265 ghash_mul_next (gd, u8x16_reflect (d[2]), kd->Hi[5]);
267 /* AES rounds 6 and 7 */
268 aesni_gcm_enc_round (r, rk[6], 4);
269 aesni_gcm_enc_round (r, rk[7], 4);
271 /* GHASH multiply block 3 */
272 ghash_mul_next (gd, u8x16_reflect (d[3]), kd->Hi[4]);
274 /* AES rounds 8 and 9 */
275 aesni_gcm_enc_round (r, rk[8], 4);
276 aesni_gcm_enc_round (r, rk[9], 4);
278 /* load 4 blocks of data - encrypt round */
280 aesni_gcm_load (d, inv, 4, 0);
282 /* AES last round(s) */
283 aesni_gcm_enc_last_round (r, d, rk, rounds, 4);
285 /* store 4 blocks of data */
286 aesni_gcm_store (d, outv, 4, 0);
288 /* load next 4 blocks of data data - decrypt round */
290 aesni_gcm_load (d, inv + 4, 4, 0);
292 /* GHASH multiply block 4 */
293 ghash_mul_next (gd, u8x16_reflect (d[0]), kd->Hi[3]);
295 /* AES rounds 0, 1 and 2 */
296 aesni_gcm_enc_first_round (r, Y, ctr, rk[0], 4);
297 aesni_gcm_enc_round (r, rk[1], 4);
298 aesni_gcm_enc_round (r, rk[2], 4);
300 /* GHASH multiply block 5 */
301 ghash_mul_next (gd, u8x16_reflect (d[1]), kd->Hi[2]);
303 /* AES rounds 3 and 4 */
304 aesni_gcm_enc_round (r, rk[3], 4);
305 aesni_gcm_enc_round (r, rk[4], 4);
307 /* GHASH multiply block 6 */
308 ghash_mul_next (gd, u8x16_reflect (d[2]), kd->Hi[1]);
310 /* AES rounds 5 and 6 */
311 aesni_gcm_enc_round (r, rk[5], 4);
312 aesni_gcm_enc_round (r, rk[6], 4);
314 /* GHASH multiply block 7 */
315 ghash_mul_next (gd, u8x16_reflect (d[3]), kd->Hi[0]);
317 /* AES rounds 7 and 8 */
318 aesni_gcm_enc_round (r, rk[7], 4);
319 aesni_gcm_enc_round (r, rk[8], 4);
321 /* GHASH reduce 1st step */
325 aesni_gcm_enc_round (r, rk[9], 4);
327 /* load data - encrypt round */
329 aesni_gcm_load (d, inv + 4, 4, 0);
331 /* GHASH reduce 2nd step */
334 /* AES last round(s) */
335 aesni_gcm_enc_last_round (r, d, rk, rounds, 4);
338 aesni_gcm_store (d, outv + 4, 4, 0);
340 /* GHASH final step */
341 return ghash_final (gd);
344 static_always_inline u8x16
345 aesni_gcm_ghash_last (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
346 int n_blocks, int n_bytes)
348 ghash_data_t _gd, *gd = &_gd;
351 d[n_blocks - 1] = aes_byte_mask (d[n_blocks - 1], n_bytes);
353 ghash_mul_first (gd, u8x16_reflect (d[0]) ^ T, kd->Hi[n_blocks - 1]);
355 ghash_mul_next (gd, u8x16_reflect (d[1]), kd->Hi[n_blocks - 2]);
357 ghash_mul_next (gd, u8x16_reflect (d[2]), kd->Hi[n_blocks - 3]);
359 ghash_mul_next (gd, u8x16_reflect (d[3]), kd->Hi[n_blocks - 4]);
362 return ghash_final (gd);
366 static_always_inline u8x16
367 aesni_gcm_enc (u8x16 T, aes_gcm_key_data_t * kd, u32x4 Y, u8x16u * inv,
368 u8x16u * outv, u32 n_left, int rounds)
381 aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
382 /* with_ghash */ 0, /* is_encrypt */ 1);
383 return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
385 else if (n_left > 32)
388 aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
389 /* with_ghash */ 0, /* is_encrypt */ 1);
390 return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
392 else if (n_left > 16)
395 aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
396 /* with_ghash */ 0, /* is_encrypt */ 1);
397 return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
402 aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
403 /* with_ghash */ 0, /* is_encrypt */ 1);
404 return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
408 aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
409 /* with_ghash */ 0, /* is_encrypt */ 1);
416 while (n_left >= 128)
418 T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
429 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
430 /* with_ghash */ 1, /* is_encrypt */ 1);
439 return aesni_gcm_ghash_last (T, kd, d, 4, 0);
444 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
445 /* with_ghash */ 1, /* is_encrypt */ 1);
446 return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
452 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
453 /* with_ghash */ 1, /* is_encrypt */ 1);
454 return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
460 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
461 /* with_ghash */ 1, /* is_encrypt */ 1);
462 return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
466 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
467 /* with_ghash */ 1, /* is_encrypt */ 1);
468 return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
471 static_always_inline u8x16
472 aesni_gcm_dec (u8x16 T, aes_gcm_key_data_t * kd, u32x4 Y, u8x16u * inv,
473 u8x16u * outv, u32 n_left, int rounds)
478 while (n_left >= 128)
480 T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
491 T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0, 1, 0);
503 return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4,
505 /* with_ghash */ 1, /* is_encrypt */ 0);
508 return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3,
510 /* with_ghash */ 1, /* is_encrypt */ 0);
513 return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2,
515 /* with_ghash */ 1, /* is_encrypt */ 0);
517 return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
518 /* with_ghash */ 1, /* is_encrypt */ 0);
521 static_always_inline int
522 aes_gcm (u8x16u * in, u8x16u * out, u8x16u * addt, u8x16u * iv, u8x16u * tag,
523 u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t * kd,
524 int aes_rounds, int is_encrypt)
529 ghash_data_t _gd, *gd = &_gd;
531 clib_prefetch_load (iv);
532 clib_prefetch_load (in);
533 clib_prefetch_load (in + 4);
535 /* calculate ghash for AAD - optimized for ipsec common cases */
537 T = aesni_gcm_ghash (T, kd, addt, 8);
538 else if (aad_bytes == 12)
539 T = aesni_gcm_ghash (T, kd, addt, 12);
541 T = aesni_gcm_ghash (T, kd, addt, aad_bytes);
543 /* initalize counter */
544 Y0 = (u32x4) aes_load_partial (iv, 12);
545 Y0[3] = clib_host_to_net_u32 (1);
547 /* ghash and encrypt/edcrypt */
549 T = aesni_gcm_enc (T, kd, Y0, in, out, data_bytes, aes_rounds);
551 T = aesni_gcm_dec (T, kd, Y0, in, out, data_bytes, aes_rounds);
553 clib_prefetch_load (tag);
555 /* Finalize ghash - data bytes and aad bytes converted to bits */
557 r = (u8x16) ((u64x2) {data_bytes, aad_bytes} << 3);
560 /* interleaved computation of final ghash and E(Y0, k) */
561 ghash_mul_first (gd, r ^ T, kd->Hi[0]);
562 r = kd->Ke[0] ^ (u8x16) Y0;
563 for (i = 1; i < 5; i += 1)
564 r = aes_enc_round (r, kd->Ke[i]);
567 for (; i < 9; i += 1)
568 r = aes_enc_round (r, kd->Ke[i]);
569 T = ghash_final (gd);
570 for (; i < aes_rounds; i += 1)
571 r = aes_enc_round (r, kd->Ke[i]);
572 r = aes_enc_last_round (r, kd->Ke[aes_rounds]);
573 T = u8x16_reflect (T) ^ r;
575 /* tag_len 16 -> 0 */
582 aes_store_partial (tag, T, (1 << tag_len) - 1);
589 u16 tag_mask = tag_len ? (1 << tag_len) - 1 : 0xffff;
590 if ((u8x16_msb_mask (tag[0] == T) & tag_mask) != tag_mask)
596 static_always_inline u32
597 aesni_ops_enc_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
598 u32 n_ops, aes_key_size_t ks)
600 crypto_native_main_t *cm = &crypto_native_main;
601 vnet_crypto_op_t *op = ops[0];
602 aes_gcm_key_data_t *kd;
607 kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
608 aes_gcm ((u8x16u *) op->src, (u8x16u *) op->dst, (u8x16u *) op->aad,
609 (u8x16u *) op->iv, (u8x16u *) op->tag, op->len, op->aad_len,
610 op->tag_len, kd, AES_KEY_ROUNDS (ks), /* is_encrypt */ 1);
611 op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
622 static_always_inline u32
623 aesni_ops_dec_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
624 u32 n_ops, aes_key_size_t ks)
626 crypto_native_main_t *cm = &crypto_native_main;
627 vnet_crypto_op_t *op = ops[0];
628 aes_gcm_key_data_t *kd;
633 kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
634 rv = aes_gcm ((u8x16u *) op->src, (u8x16u *) op->dst, (u8x16u *) op->aad,
635 (u8x16u *) op->iv, (u8x16u *) op->tag, op->len,
636 op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
641 op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
645 op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
658 static_always_inline void *
659 aesni_gcm_key_exp (vnet_crypto_key_t * key, aes_key_size_t ks)
661 aes_gcm_key_data_t *kd;
664 kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
667 aes_key_expand ((u8x16 *) kd->Ke, key->data, ks);
669 /* pre-calculate H */
670 H = aes_encrypt_block (u8x16_splat (0), kd->Ke, ks);
671 H = u8x16_reflect (H);
672 ghash_precompute (H, (u8x16 *) kd->Hi, 8);
676 #define foreach_aesni_gcm_handler_type _(128) _(192) _(256)
679 static u32 aesni_ops_dec_aes_gcm_##x \
680 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
681 { return aesni_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
682 static u32 aesni_ops_enc_aes_gcm_##x \
683 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
684 { return aesni_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
685 static void * aesni_gcm_key_exp_##x (vnet_crypto_key_t *key) \
686 { return aesni_gcm_key_exp (key, AES_KEY_##x); }
688 foreach_aesni_gcm_handler_type;
693 crypto_native_aes_gcm_init_vaes (vlib_main_t * vm)
695 crypto_native_aes_gcm_init_avx512 (vlib_main_t * vm)
697 crypto_native_aes_gcm_init_avx2 (vlib_main_t * vm)
699 crypto_native_aes_gcm_init_neon (vlib_main_t * vm)
701 crypto_native_aes_gcm_init_sse42 (vlib_main_t * vm)
704 crypto_native_main_t *cm = &crypto_native_main;
707 vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
708 VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \
709 aesni_ops_enc_aes_gcm_##x); \
710 vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
711 VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \
712 aesni_ops_dec_aes_gcm_##x); \
713 cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aesni_gcm_key_exp_##x;
714 foreach_aesni_gcm_handler_type;
720 * fd.io coding-style-patch-verification: ON
723 * eval: (c-set-style "gnu")