From d5023a72ddabe7e9a8698ad57c82d6be117fff73 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 22 May 2019 15:11:18 +0200 Subject: [PATCH] crypto_ia32: multiarch Change-Id: Iead43a2b524b735a2069e611d899cd41d3a8efdc Signed-off-by: Damjan Marion --- src/plugins/crypto_ia32/CMakeLists.txt | 22 ++++++++++---- src/plugins/crypto_ia32/aes_cbc.c | 41 ++++++++++++++++++++++----- src/plugins/crypto_ia32/aesni.h | 6 ---- src/plugins/crypto_ia32/crypto_ia32.h | 7 ++++- src/plugins/crypto_ia32/main.c | 52 +++++++++------------------------- 5 files changed, 70 insertions(+), 58 deletions(-) diff --git a/src/plugins/crypto_ia32/CMakeLists.txt b/src/plugins/crypto_ia32/CMakeLists.txt index a100cdbb681..b01c57b9f6f 100644 --- a/src/plugins/crypto_ia32/CMakeLists.txt +++ b/src/plugins/crypto_ia32/CMakeLists.txt @@ -15,10 +15,20 @@ if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*") return() endif() -add_vpp_plugin(crypto_ia32 - SOURCES - aes_cbc.c - main.c -) +add_vpp_plugin(crypto_ia32 SOURCES main.c) -target_compile_options(crypto_ia32_plugin PRIVATE "-march=silvermont") +list(APPEND VARIANTS "sse42\;-march=silvermont") +list(APPEND VARIANTS "avx2\;-march=core-avx2") +if(compiler_flag_march_skylake_avx512) + list(APPEND VARIANTS "avx512\;-march=skylake-avx512") +endif() + +foreach(VARIANT ${VARIANTS}) + list(GET VARIANT 0 v) + list(GET VARIANT 1 f) + set(l crypto_ia32_${v}) + add_library(${l} OBJECT aes_cbc.c) + set_target_properties(${l} PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_options(${l} PUBLIC ${f} -O3 -Wall -fno-common) + target_sources(crypto_ia32_plugin PRIVATE $) +endforeach() diff --git a/src/plugins/crypto_ia32/aes_cbc.c b/src/plugins/crypto_ia32/aes_cbc.c index 7befdd2b4e6..2c08aa207ac 100644 --- a/src/plugins/crypto_ia32/aes_cbc.c +++ b/src/plugins/crypto_ia32/aes_cbc.c @@ -22,6 +22,12 @@ #include #include +typedef struct +{ + __m128i encrypt_key[15]; + __m128i decrypt_key[15]; +} aes_cbc_key_data_t; + static_always_inline void aes_cbc_dec (__m128i * k, u8 * src, u8 * dst, u8 * iv, int count, aesni_key_size_t rounds) @@ -98,7 +104,8 @@ aesni_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], u8 *src[4] = { }; u8 *dst[4] = { }; vnet_crypto_key_index_t key_index[4] = { ~0, ~0, ~0, ~0 }; - u32x4 dummy_mask, len = { }; + u32x4 dummy_mask = { }; + u32x4 len = { }; u32 i, j, count, n_left = n_ops; __m128i r[4] = { }, k[4][rounds + 1]; @@ -129,9 +136,9 @@ more: dummy_mask[i] = ~0; if (key_index[i] != ops[0]->key_index) { - aesni_key_data_t *kd; + aes_cbc_key_data_t *kd; key_index[i] = ops[0]->key_index; - kd = (aesni_key_data_t *) cm->key_data[key_index[i]]; + kd = (aes_cbc_key_data_t *) cm->key_data[key_index[i]]; clib_memcpy_fast (k[i], kd->encrypt_key, (rounds + 1) * sizeof (__m128i)); } @@ -194,7 +201,7 @@ aesni_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[], crypto_ia32_main_t *cm = &crypto_ia32_main; int rounds = AESNI_KEY_ROUNDS (ks); vnet_crypto_op_t *op = ops[0]; - aesni_key_data_t *kd = (aesni_key_data_t *) cm->key_data[op->key_index]; + aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index]; u32 n_left = n_ops; ASSERT (n_ops >= 1); @@ -206,13 +213,24 @@ decrypt: if (--n_left) { op += 1; - kd = (aesni_key_data_t *) cm->key_data[op->key_index]; + kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index]; goto decrypt; } return n_ops; } +static_always_inline void * +aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks) +{ + aes_cbc_key_data_t *kd; + kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES); + aes_key_expand (kd->encrypt_key, key->data, ks); + aes_key_expand (kd->decrypt_key, key->data, ks); + aes_key_enc_to_dec (kd->decrypt_key, ks); + return kd; +} + #define foreach_aesni_cbc_handler_type _(128) _(192) _(256) #define _(x) \ @@ -222,6 +240,8 @@ static u32 aesni_ops_dec_aes_cbc_##x \ static u32 aesni_ops_enc_aes_cbc_##x \ (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \ { return aesni_ops_enc_aes_cbc (vm, ops, n_ops, AESNI_KEY_##x); } \ +static void * aesni_cbc_key_exp_##x (vnet_crypto_key_t *key) \ +{ return aesni_cbc_key_exp (key, AESNI_KEY_##x); } foreach_aesni_cbc_handler_type; #undef _ @@ -229,7 +249,13 @@ foreach_aesni_cbc_handler_type; #include clib_error_t * -crypto_ia32_aesni_cbc_init (vlib_main_t * vm) +#ifdef __AVX512F__ +crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm) +#elif __AVX2__ +crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm) +#else +crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm) +#endif { crypto_ia32_main_t *cm = &crypto_ia32_main; crypto_ia32_per_thread_data_t *ptd; @@ -260,7 +286,8 @@ crypto_ia32_aesni_cbc_init (vlib_main_t * vm) aesni_ops_enc_aes_cbc_##x); \ vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \ VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \ - aesni_ops_dec_aes_cbc_##x); + aesni_ops_dec_aes_cbc_##x); \ + cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aesni_cbc_key_exp_##x; foreach_aesni_cbc_handler_type; #undef _ diff --git a/src/plugins/crypto_ia32/aesni.h b/src/plugins/crypto_ia32/aesni.h index 80e30e51b33..ceb28451732 100644 --- a/src/plugins/crypto_ia32/aesni.h +++ b/src/plugins/crypto_ia32/aesni.h @@ -19,12 +19,6 @@ #define __aesni_h__ -typedef struct -{ - __m128i encrypt_key[15]; - __m128i decrypt_key[15]; -} aesni_key_data_t; - typedef enum { AESNI_KEY_128 = 0, diff --git a/src/plugins/crypto_ia32/crypto_ia32.h b/src/plugins/crypto_ia32/crypto_ia32.h index f5f09dfd499..7acf0741ec6 100644 --- a/src/plugins/crypto_ia32/crypto_ia32.h +++ b/src/plugins/crypto_ia32/crypto_ia32.h @@ -18,6 +18,8 @@ #ifndef __crypto_ia32_h__ #define __crypto_ia32_h__ +typedef void *(crypto_ia32_key_fn_t) (vnet_crypto_key_t * key); + typedef struct { __m128i cbc_iv[4]; @@ -27,12 +29,15 @@ typedef struct { u32 crypto_engine_index; crypto_ia32_per_thread_data_t *per_thread_data; + crypto_ia32_key_fn_t *key_fn[VNET_CRYPTO_N_ALGS]; void **key_data; } crypto_ia32_main_t; extern crypto_ia32_main_t crypto_ia32_main; -clib_error_t *crypto_ia32_aesni_cbc_init (vlib_main_t * vm); +clib_error_t *crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm); +clib_error_t *crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm); +clib_error_t *crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm); #endif /* __crypto_ia32_h__ */ diff --git a/src/plugins/crypto_ia32/main.c b/src/plugins/crypto_ia32/main.c index a02fc38f99e..b31c9019495 100644 --- a/src/plugins/crypto_ia32/main.c +++ b/src/plugins/crypto_ia32/main.c @@ -19,7 +19,6 @@ #include #include #include -#include crypto_ia32_main_t crypto_ia32_main; @@ -29,17 +28,9 @@ crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, { vnet_crypto_key_t *key = vnet_crypto_get_key (idx); crypto_ia32_main_t *cm = &crypto_ia32_main; - aesni_key_data_t *kd; - switch (key->alg) - { - case VNET_CRYPTO_ALG_AES_128_CBC: - case VNET_CRYPTO_ALG_AES_192_CBC: - case VNET_CRYPTO_ALG_AES_256_CBC: - break; - default: - return; - } + if (cm->key_fn[key->alg] == 0) + return; if (kop == VNET_CRYPTO_KEY_OP_DEL) { @@ -65,31 +56,7 @@ crypto_ia32_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, clib_mem_free (cm->key_data[idx]); } - kd = cm->key_data[idx] = clib_mem_alloc_aligned (sizeof (aesni_key_data_t), - CLIB_CACHE_LINE_BYTES); - - /* ADD or MODIFY */ - switch (key->alg) - { - case VNET_CRYPTO_ALG_AES_128_CBC: - aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_128); - aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_128); - aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_128); - break; - case VNET_CRYPTO_ALG_AES_192_CBC: - aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_192); - aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_192); - aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_192); - break; - case VNET_CRYPTO_ALG_AES_256_CBC: - aes_key_expand (kd->encrypt_key, key->data, AESNI_KEY_256); - aes_key_expand (kd->decrypt_key, key->data, AESNI_KEY_256); - aes_key_enc_to_dec (kd->decrypt_key, AESNI_KEY_256); - break; - default: - break; - } - return; + cm->key_data[idx] = cm->key_fn[key->alg] (key); } clib_error_t * @@ -99,6 +66,9 @@ crypto_ia32_init (vlib_main_t * vm) vlib_thread_main_t *tm = vlib_get_thread_main (); clib_error_t *error = 0; + if (clib_cpu_supports_x86_aes () == 0) + return 0; + vec_validate_aligned (cm->per_thread_data, tm->n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); @@ -106,8 +76,14 @@ crypto_ia32_init (vlib_main_t * vm) vnet_crypto_register_engine (vm, "ia32", 100, "Intel IA32 ISA Optimized Crypto"); - if (clib_cpu_supports_x86_aes () && - (error = crypto_ia32_aesni_cbc_init (vm))) + if (clib_cpu_supports_avx512f ()) + error = crypto_ia32_aesni_cbc_init_avx512 (vm); + else if (clib_cpu_supports_avx2 ()) + error = crypto_ia32_aesni_cbc_init_avx2 (vm); + else + error = crypto_ia32_aesni_cbc_init_sse42 (vm); + + if (error) goto error; vnet_crypto_register_key_handler (vm, cm->crypto_engine_index, -- 2.16.6