2 *------------------------------------------------------------------
3 * Copyright (c) 2019 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <crypto_native/crypto_native.h>
22 #include <crypto_native/aes.h>
23 #include <crypto_native/ghash.h>
25 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
26 #pragma GCC optimize ("O3")
32 /* pre-calculated hash key values */
33 const u8x16 Hi[NUM_HI];
34 /* extracted AES key */
46 AES_GCM_F_WITH_GHASH = (1 << 0),
47 AES_GCM_F_LAST_ROUND = (1 << 1),
48 AES_GCM_F_ENCRYPT = (1 << 2),
49 AES_GCM_F_DECRYPT = (1 << 3),
52 static const u32x4 ctr_inv_1 = { 0, 0, 0, 1 << 24 };
56 static_always_inline void
57 aes_gcm_enc_first_round (u8x16 * r, aes_gcm_counter_t * ctr, u8x16 k,
60 if (PREDICT_TRUE ((u8) ctr->counter < (256 - 2 * n_blocks)))
62 for (int i = 0; i < n_blocks; i++)
64 r[i] = k ^ (u8x16) ctr->Y;
67 ctr->counter += n_blocks;
71 for (int i = 0; i < n_blocks; i++)
73 r[i] = k ^ (u8x16) ctr->Y;
75 ctr->Y[3] = clib_host_to_net_u32 (ctr->counter + 1);
80 static_always_inline void
81 aes_gcm_enc_round (u8x16 * r, u8x16 k, int n_blocks)
83 for (int i = 0; i < n_blocks; i++)
84 r[i] = aes_enc_round (r[i], k);
87 static_always_inline void
88 aes_gcm_enc_last_round (u8x16 * r, u8x16 * d, u8x16 const *k,
89 int rounds, int n_blocks)
92 /* additional ronuds for AES-192 and AES-256 */
93 for (int i = 10; i < rounds; i++)
94 aes_gcm_enc_round (r, k[i], n_blocks);
96 for (int i = 0; i < n_blocks; i++)
97 d[i] ^= aes_enc_last_round (r[i], k[rounds]);
100 static_always_inline u8x16
101 aes_gcm_ghash_blocks (u8x16 T, aes_gcm_key_data_t * kd,
102 u8x16u * in, int n_blocks)
104 ghash_data_t _gd, *gd = &_gd;
105 u8x16 *Hi = (u8x16 *) kd->Hi + NUM_HI - n_blocks;
106 ghash_mul_first (gd, u8x16_reflect (in[0]) ^ T, Hi[0]);
107 for (int i = 1; i < n_blocks; i++)
108 ghash_mul_next (gd, u8x16_reflect ((in[i])), Hi[i]);
111 return ghash_final (gd);
114 static_always_inline u8x16
115 aes_gcm_ghash (u8x16 T, aes_gcm_key_data_t * kd, u8x16u * in, u32 n_left)
118 while (n_left >= 128)
120 T = aes_gcm_ghash_blocks (T, kd, in, 8);
127 T = aes_gcm_ghash_blocks (T, kd, in, 4);
134 T = aes_gcm_ghash_blocks (T, kd, in, 2);
141 T = aes_gcm_ghash_blocks (T, kd, in, 1);
148 u8x16 r = aes_load_partial (in, n_left);
149 T = ghash_mul (u8x16_reflect (r) ^ T, kd->Hi[NUM_HI - 1]);
154 static_always_inline u8x16
155 aes_gcm_calc (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
156 aes_gcm_counter_t * ctr, u8x16u * inv, u8x16u * outv,
157 int rounds, int n, int last_block_bytes, aes_gcm_flags_t f)
160 ghash_data_t _gd = { }, *gd = &_gd;
161 const u8x16 *rk = (u8x16 *) kd->Ke;
162 int ghash_blocks = (f & AES_GCM_F_ENCRYPT) ? 4 : n, gc = 1;
163 u8x16 *Hi = (u8x16 *) kd->Hi + NUM_HI - ghash_blocks;
165 clib_prefetch_load (inv + 4);
167 /* AES rounds 0 and 1 */
168 aes_gcm_enc_first_round (r, ctr, rk[0], n);
169 aes_gcm_enc_round (r, rk[1], n);
171 /* load data - decrypt round */
172 if (f & AES_GCM_F_DECRYPT)
174 for (int i = 0; i < n - ((f & AES_GCM_F_LAST_ROUND) != 0); i++)
177 if (f & AES_GCM_F_LAST_ROUND)
178 d[n - 1] = aes_load_partial (inv + n - 1, last_block_bytes);
181 /* GHASH multiply block 1 */
182 if (f & AES_GCM_F_WITH_GHASH)
183 ghash_mul_first (gd, u8x16_reflect (d[0]) ^ T, Hi[0]);
185 /* AES rounds 2 and 3 */
186 aes_gcm_enc_round (r, rk[2], n);
187 aes_gcm_enc_round (r, rk[3], n);
189 /* GHASH multiply block 2 */
190 if ((f & AES_GCM_F_WITH_GHASH) && gc++ < ghash_blocks)
191 ghash_mul_next (gd, u8x16_reflect (d[1]), Hi[1]);
193 /* AES rounds 4 and 5 */
194 aes_gcm_enc_round (r, rk[4], n);
195 aes_gcm_enc_round (r, rk[5], n);
197 /* GHASH multiply block 3 */
198 if ((f & AES_GCM_F_WITH_GHASH) && gc++ < ghash_blocks)
199 ghash_mul_next (gd, u8x16_reflect (d[2]), Hi[2]);
201 /* AES rounds 6 and 7 */
202 aes_gcm_enc_round (r, rk[6], n);
203 aes_gcm_enc_round (r, rk[7], n);
205 /* GHASH multiply block 4 */
206 if ((f & AES_GCM_F_WITH_GHASH) && gc++ < ghash_blocks)
207 ghash_mul_next (gd, u8x16_reflect (d[3]), Hi[3]);
209 /* AES rounds 8 and 9 */
210 aes_gcm_enc_round (r, rk[8], n);
211 aes_gcm_enc_round (r, rk[9], n);
213 /* GHASH reduce 1st step */
214 if (f & AES_GCM_F_WITH_GHASH)
217 /* load data - encrypt round */
218 if (f & AES_GCM_F_ENCRYPT)
220 for (int i = 0; i < n - ((f & AES_GCM_F_LAST_ROUND) != 0); i++)
223 if (f & AES_GCM_F_LAST_ROUND)
224 d[n - 1] = aes_load_partial (inv + n - 1, last_block_bytes);
227 /* GHASH reduce 2nd step */
228 if (f & AES_GCM_F_WITH_GHASH)
231 /* AES last round(s) */
232 aes_gcm_enc_last_round (r, d, rk, rounds, n);
235 for (int i = 0; i < n - ((f & AES_GCM_F_LAST_ROUND) != 0); i++)
238 if (f & AES_GCM_F_LAST_ROUND)
239 aes_store_partial (outv + n - 1, d[n - 1], last_block_bytes);
241 /* GHASH final step */
242 if (f & AES_GCM_F_WITH_GHASH)
243 T = ghash_final (gd);
248 static_always_inline u8x16
249 aes_gcm_calc_double (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
250 aes_gcm_counter_t * ctr, u8x16u * inv, u8x16u * outv,
251 int rounds, aes_gcm_flags_t f)
254 ghash_data_t _gd, *gd = &_gd;
255 const u8x16 *rk = (u8x16 *) kd->Ke;
256 u8x16 *Hi = (u8x16 *) kd->Hi + NUM_HI - 8;
258 /* AES rounds 0 and 1 */
259 aes_gcm_enc_first_round (r, ctr, rk[0], 4);
260 aes_gcm_enc_round (r, rk[1], 4);
262 /* load 4 blocks of data - decrypt round */
263 if (f & AES_GCM_F_DECRYPT)
271 /* GHASH multiply block 0 */
272 ghash_mul_first (gd, u8x16_reflect (d[0]) ^ T, Hi[0]);
274 /* AES rounds 2 and 3 */
275 aes_gcm_enc_round (r, rk[2], 4);
276 aes_gcm_enc_round (r, rk[3], 4);
278 /* GHASH multiply block 1 */
279 ghash_mul_next (gd, u8x16_reflect (d[1]), Hi[1]);
281 /* AES rounds 4 and 5 */
282 aes_gcm_enc_round (r, rk[4], 4);
283 aes_gcm_enc_round (r, rk[5], 4);
285 /* GHASH multiply block 2 */
286 ghash_mul_next (gd, u8x16_reflect (d[2]), Hi[2]);
288 /* AES rounds 6 and 7 */
289 aes_gcm_enc_round (r, rk[6], 4);
290 aes_gcm_enc_round (r, rk[7], 4);
292 /* GHASH multiply block 3 */
293 ghash_mul_next (gd, u8x16_reflect (d[3]), Hi[3]);
295 /* AES rounds 8 and 9 */
296 aes_gcm_enc_round (r, rk[8], 4);
297 aes_gcm_enc_round (r, rk[9], 4);
299 /* load 4 blocks of data - encrypt round */
300 if (f & AES_GCM_F_ENCRYPT)
308 /* AES last round(s) */
309 aes_gcm_enc_last_round (r, d, rk, rounds, 4);
311 /* store 4 blocks of data */
317 /* load next 4 blocks of data data - decrypt round */
318 if (f & AES_GCM_F_DECRYPT)
326 /* GHASH multiply block 4 */
327 ghash_mul_next (gd, u8x16_reflect (d[0]), Hi[4]);
329 /* AES rounds 0, 1 and 2 */
330 aes_gcm_enc_first_round (r, ctr, rk[0], 4);
331 aes_gcm_enc_round (r, rk[1], 4);
332 aes_gcm_enc_round (r, rk[2], 4);
334 /* GHASH multiply block 5 */
335 ghash_mul_next (gd, u8x16_reflect (d[1]), Hi[5]);
337 /* AES rounds 3 and 4 */
338 aes_gcm_enc_round (r, rk[3], 4);
339 aes_gcm_enc_round (r, rk[4], 4);
341 /* GHASH multiply block 6 */
342 ghash_mul_next (gd, u8x16_reflect (d[2]), Hi[6]);
344 /* AES rounds 5 and 6 */
345 aes_gcm_enc_round (r, rk[5], 4);
346 aes_gcm_enc_round (r, rk[6], 4);
348 /* GHASH multiply block 7 */
349 ghash_mul_next (gd, u8x16_reflect (d[3]), Hi[7]);
351 /* AES rounds 7 and 8 */
352 aes_gcm_enc_round (r, rk[7], 4);
353 aes_gcm_enc_round (r, rk[8], 4);
355 /* GHASH reduce 1st step */
359 aes_gcm_enc_round (r, rk[9], 4);
361 /* load data - encrypt round */
362 if (f & AES_GCM_F_ENCRYPT)
370 /* GHASH reduce 2nd step */
373 /* AES last round(s) */
374 aes_gcm_enc_last_round (r, d, rk, rounds, 4);
382 /* GHASH final step */
383 return ghash_final (gd);
386 static_always_inline u8x16
387 aes_gcm_ghash_last (u8x16 T, aes_gcm_key_data_t * kd, u8x16 * d,
388 int n_blocks, int n_bytes)
390 ghash_data_t _gd, *gd = &_gd;
391 u8x16 *Hi = (u8x16 *) kd->Hi + NUM_HI - n_blocks;
394 d[n_blocks - 1] = aes_byte_mask (d[n_blocks - 1], n_bytes);
396 ghash_mul_first (gd, u8x16_reflect (d[0]) ^ T, Hi[0]);
398 ghash_mul_next (gd, u8x16_reflect (d[1]), Hi[1]);
400 ghash_mul_next (gd, u8x16_reflect (d[2]), Hi[2]);
402 ghash_mul_next (gd, u8x16_reflect (d[3]), Hi[3]);
405 return ghash_final (gd);
409 static_always_inline u8x16
410 aes_gcm_enc (u8x16 T, aes_gcm_key_data_t * kd, aes_gcm_counter_t * ctr,
411 u8x16u * inv, u8x16u * outv, u32 n_left, int rounds)
414 aes_gcm_flags_t f = AES_GCM_F_ENCRYPT;
421 f |= AES_GCM_F_LAST_ROUND;
425 aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, n_left, f);
426 return aes_gcm_ghash_last (T, kd, d, 4, n_left);
428 else if (n_left > 32)
431 aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 3, n_left, f);
432 return aes_gcm_ghash_last (T, kd, d, 3, n_left);
434 else if (n_left > 16)
437 aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 2, n_left, f);
438 return aes_gcm_ghash_last (T, kd, d, 2, n_left);
442 aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 1, n_left, f);
443 return aes_gcm_ghash_last (T, kd, d, 1, n_left);
447 aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, 0, f);
454 f |= AES_GCM_F_WITH_GHASH;
456 while (n_left >= 128)
458 T = aes_gcm_calc_double (T, kd, d, ctr, inv, outv, rounds, f);
468 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, 0, f);
477 return aes_gcm_ghash_last (T, kd, d, 4, 0);
479 f |= AES_GCM_F_LAST_ROUND;
484 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, n_left, f);
485 return aes_gcm_ghash_last (T, kd, d, 4, n_left);
491 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 3, n_left, f);
492 return aes_gcm_ghash_last (T, kd, d, 3, n_left);
498 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 2, n_left, f);
499 return aes_gcm_ghash_last (T, kd, d, 2, n_left);
502 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 1, n_left, f);
503 return aes_gcm_ghash_last (T, kd, d, 1, n_left);
506 static_always_inline u8x16
507 aes_gcm_dec (u8x16 T, aes_gcm_key_data_t * kd, aes_gcm_counter_t * ctr,
508 u8x16u * inv, u8x16u * outv, u32 n_left, int rounds)
511 aes_gcm_flags_t f = AES_GCM_F_WITH_GHASH | AES_GCM_F_DECRYPT;
513 while (n_left >= 128)
515 T = aes_gcm_calc_double (T, kd, d, ctr, inv, outv, rounds, f);
525 T = aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, 0, f);
536 f |= AES_GCM_F_LAST_ROUND;
539 return aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 4, n_left - 48, f);
542 return aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 3, n_left - 32, f);
545 return aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 2, n_left - 16, f);
547 return aes_gcm_calc (T, kd, d, ctr, inv, outv, rounds, 1, n_left, f);
550 static_always_inline int
551 aes_gcm (u8x16u * in, u8x16u * out, u8x16u * addt, u8x16u * iv, u8x16u * tag,
552 u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t * kd,
553 int aes_rounds, int is_encrypt)
558 ghash_data_t _gd, *gd = &_gd;
559 aes_gcm_counter_t _ctr, *ctr = &_ctr;
561 clib_prefetch_load (iv);
562 clib_prefetch_load (in);
563 clib_prefetch_load (in + 4);
565 /* calculate ghash for AAD - optimized for ipsec common cases */
567 T = aes_gcm_ghash (T, kd, addt, 8);
568 else if (aad_bytes == 12)
569 T = aes_gcm_ghash (T, kd, addt, 12);
571 T = aes_gcm_ghash (T, kd, addt, aad_bytes);
573 /* initalize counter */
575 Y0 = (u32x4) aes_load_partial (iv, 12) + ctr_inv_1;
576 ctr->Y = Y0 + ctr_inv_1;
578 /* ghash and encrypt/edcrypt */
580 T = aes_gcm_enc (T, kd, ctr, in, out, data_bytes, aes_rounds);
582 T = aes_gcm_dec (T, kd, ctr, in, out, data_bytes, aes_rounds);
584 clib_prefetch_load (tag);
586 /* Finalize ghash - data bytes and aad bytes converted to bits */
588 r = (u8x16) ((u64x2) {data_bytes, aad_bytes} << 3);
591 /* interleaved computation of final ghash and E(Y0, k) */
592 ghash_mul_first (gd, r ^ T, kd->Hi[NUM_HI - 1]);
593 r = kd->Ke[0] ^ (u8x16) Y0;
594 for (i = 1; i < 5; i += 1)
595 r = aes_enc_round (r, kd->Ke[i]);
598 for (; i < 9; i += 1)
599 r = aes_enc_round (r, kd->Ke[i]);
600 T = ghash_final (gd);
601 for (; i < aes_rounds; i += 1)
602 r = aes_enc_round (r, kd->Ke[i]);
603 r = aes_enc_last_round (r, kd->Ke[aes_rounds]);
604 T = u8x16_reflect (T) ^ r;
606 /* tag_len 16 -> 0 */
613 aes_store_partial (tag, T, tag_len);
620 u16 tag_mask = tag_len ? (1 << tag_len) - 1 : 0xffff;
621 if ((u8x16_msb_mask (tag[0] == T) & tag_mask) != tag_mask)
627 static_always_inline u32
628 aes_ops_enc_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[],
629 u32 n_ops, aes_key_size_t ks)
631 crypto_native_main_t *cm = &crypto_native_main;
632 vnet_crypto_op_t *op = ops[0];
633 aes_gcm_key_data_t *kd;
638 kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
639 aes_gcm ((u8x16u *) op->src, (u8x16u *) op->dst, (u8x16u *) op->aad,
640 (u8x16u *) op->iv, (u8x16u *) op->tag, op->len, op->aad_len,
641 op->tag_len, kd, AES_KEY_ROUNDS (ks), /* is_encrypt */ 1);
642 op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
653 static_always_inline u32
654 aes_ops_dec_aes_gcm (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops,
657 crypto_native_main_t *cm = &crypto_native_main;
658 vnet_crypto_op_t *op = ops[0];
659 aes_gcm_key_data_t *kd;
664 kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
665 rv = aes_gcm ((u8x16u *) op->src, (u8x16u *) op->dst, (u8x16u *) op->aad,
666 (u8x16u *) op->iv, (u8x16u *) op->tag, op->len,
667 op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
672 op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
676 op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
689 static_always_inline void *
690 aes_gcm_key_exp (vnet_crypto_key_t * key, aes_key_size_t ks)
692 aes_gcm_key_data_t *kd;
695 kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
698 aes_key_expand ((u8x16 *) kd->Ke, key->data, ks);
700 /* pre-calculate H */
701 H = aes_encrypt_block (u8x16_splat (0), kd->Ke, ks);
702 H = u8x16_reflect (H);
703 ghash_precompute (H, (u8x16 *) kd->Hi, NUM_HI);
707 #define foreach_aes_gcm_handler_type _(128) _(192) _(256)
710 static u32 aes_ops_dec_aes_gcm_##x \
711 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
712 { return aes_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
713 static u32 aes_ops_enc_aes_gcm_##x \
714 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
715 { return aes_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x); } \
716 static void * aes_gcm_key_exp_##x (vnet_crypto_key_t *key) \
717 { return aes_gcm_key_exp (key, AES_KEY_##x); }
719 foreach_aes_gcm_handler_type;
724 crypto_native_aes_gcm_init_vaes (vlib_main_t * vm)
726 crypto_native_aes_gcm_init_avx512 (vlib_main_t * vm)
728 crypto_native_aes_gcm_init_avx2 (vlib_main_t * vm)
730 crypto_native_aes_gcm_init_neon (vlib_main_t * vm)
732 crypto_native_aes_gcm_init_sse42 (vlib_main_t * vm)
735 crypto_native_main_t *cm = &crypto_native_main;
738 vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
739 VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \
740 aes_ops_enc_aes_gcm_##x); \
741 vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
742 VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \
743 aes_ops_dec_aes_gcm_##x); \
744 cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aes_gcm_key_exp_##x;
745 foreach_aes_gcm_handler_type;
751 * fd.io coding-style-patch-verification: ON
754 * eval: (c-set-style "gnu")