Code Review
/
vpp.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
review
|
tree
raw
|
inline
| side by side
vppinfra: native AES-CTR implementation
[vpp.git]
/
src
/
vppinfra
/
crypto
/
aes_gcm.h
diff --git
a/src/vppinfra/crypto/aes_gcm.h
b/src/vppinfra/crypto/aes_gcm.h
index
3d1b220
..
5b628c8
100644
(file)
--- a/
src/vppinfra/crypto/aes_gcm.h
+++ b/
src/vppinfra/crypto/aes_gcm.h
@@
-13,47
+13,25
@@
#include <vppinfra/crypto/ghash.h>
#define NUM_HI 36
#include <vppinfra/crypto/ghash.h>
#define NUM_HI 36
-#if defined(__VAES__) && defined(__AVX512F__)
-typedef u8x64 aes_data_t;
+#if N_AES_LANES == 4
typedef u8x64u aes_ghash_t;
typedef u8x64u aes_ghash_t;
-typedef u8x64u aes_mem_t;
-typedef u32x16 aes_gcm_counter_t;
-#define N 64
-#define aes_gcm_load_partial(p, n) u8x64_load_partial ((u8 *) (p), n)
-#define aes_gcm_store_partial(v, p, n) u8x64_store_partial (v, (u8 *) (p), n)
#define aes_gcm_splat(v) u8x64_splat (v)
#define aes_gcm_splat(v) u8x64_splat (v)
-#define aes_gcm_reflect(r) u8x64_reflect_u8x16 (r)
#define aes_gcm_ghash_reduce(c) ghash4_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash4_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash4_final (&(c)->gd)
#define aes_gcm_ghash_reduce(c) ghash4_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash4_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash4_final (&(c)->gd)
-#elif defined(__VAES__)
-typedef u8x32 aes_data_t;
+#elif N_AES_LANES == 2
typedef u8x32u aes_ghash_t;
typedef u8x32u aes_ghash_t;
-typedef u8x32u aes_mem_t;
-typedef u32x8 aes_gcm_counter_t;
-#define N 32
-#define aes_gcm_load_partial(p, n) u8x32_load_partial ((u8 *) (p), n)
-#define aes_gcm_store_partial(v, p, n) u8x32_store_partial (v, (u8 *) (p), n)
#define aes_gcm_splat(v) u8x32_splat (v)
#define aes_gcm_splat(v) u8x32_splat (v)
-#define aes_gcm_reflect(r) u8x32_reflect_u8x16 (r)
#define aes_gcm_ghash_reduce(c) ghash2_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash2_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash2_final (&(c)->gd)
#else
#define aes_gcm_ghash_reduce(c) ghash2_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash2_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash2_final (&(c)->gd)
#else
-typedef u8x16 aes_data_t;
typedef u8x16 aes_ghash_t;
typedef u8x16 aes_ghash_t;
-typedef u8x16u aes_mem_t;
-typedef u32x4 aes_gcm_counter_t;
-#define N 16
-#define aes_gcm_load_partial(p, n) u8x16_load_partial ((u8 *) (p), n)
-#define aes_gcm_store_partial(v, p, n) u8x16_store_partial (v, (u8 *) (p), n)
#define aes_gcm_splat(v) u8x16_splat (v)
#define aes_gcm_splat(v) u8x16_splat (v)
-#define aes_gcm_reflect(r) u8x16_reflect (r)
#define aes_gcm_ghash_reduce(c) ghash_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash_final (&(c)->gd)
#endif
#define aes_gcm_ghash_reduce(c) ghash_reduce (&(c)->gd)
#define aes_gcm_ghash_reduce2(c) ghash_reduce2 (&(c)->gd)
#define aes_gcm_ghash_final(c) (c)->T = ghash_final (&(c)->gd)
#endif
-#define N_LANES (N / 16)
typedef enum
{
typedef enum
{
@@
-63,21
+41,12
@@
typedef enum
AES_GCM_OP_GMAC
} aes_gcm_op_t;
AES_GCM_OP_GMAC
} aes_gcm_op_t;
-typedef union
-{
- u8x16 x1;
- u8x32 x2;
- u8x64 x4;
- u8x16 lanes[4];
-} __clib_aligned (64)
-aes_gcm_expaned_key_t;
-
typedef struct
{
/* pre-calculated hash key values */
const u8x16 Hi[NUM_HI];
/* extracted AES key */
typedef struct
{
/* pre-calculated hash key values */
const u8x16 Hi[NUM_HI];
/* extracted AES key */
- const aes_
gcm_
expaned_key_t Ke[AES_KEY_ROUNDS (AES_KEY_256) + 1];
+ const aes_expaned_key_t Ke[AES_KEY_ROUNDS (AES_KEY_256) + 1];
} aes_gcm_key_data_t;
typedef struct
} aes_gcm_key_data_t;
typedef struct
@@
-95,12
+64,12
@@
typedef struct
const aes_ghash_t *next_Hi;
/* expaded keys */
const aes_ghash_t *next_Hi;
/* expaded keys */
- const aes_
gcm_
expaned_key_t *Ke;
+ const aes_expaned_key_t *Ke;
/* counter */
u32 counter;
u8x16 EY0;
/* counter */
u32 counter;
u8x16 EY0;
- aes_
gcm_
counter_t Y;
+ aes_counter_t Y;
/* ghash */
ghash_ctx_t gd;
/* ghash */
ghash_ctx_t gd;
@@
-117,39
+86,39
@@
aes_gcm_ghash_mul_first (aes_gcm_ctx_t *ctx, aes_data_t data, u32 n_lanes)
{
uword hash_offset = NUM_HI - n_lanes;
ctx->next_Hi = (aes_ghash_t *) (ctx->Hi + hash_offset);
{
uword hash_offset = NUM_HI - n_lanes;
ctx->next_Hi = (aes_ghash_t *) (ctx->Hi + hash_offset);
-#if N_LANES == 4
+#if N_
AES_
LANES == 4
u8x64 tag4 = {};
tag4 = u8x64_insert_u8x16 (tag4, ctx->T, 0);
u8x64 tag4 = {};
tag4 = u8x64_insert_u8x16 (tag4, ctx->T, 0);
- ghash4_mul_first (&ctx->gd, aes_
gcm_
reflect (data) ^ tag4, *ctx->next_Hi++);
-#elif N_LANES == 2
+ ghash4_mul_first (&ctx->gd, aes_reflect (data) ^ tag4, *ctx->next_Hi++);
+#elif N_
AES_
LANES == 2
u8x32 tag2 = {};
tag2 = u8x32_insert_lo (tag2, ctx->T);
u8x32 tag2 = {};
tag2 = u8x32_insert_lo (tag2, ctx->T);
- ghash2_mul_first (&ctx->gd, aes_
gcm_
reflect (data) ^ tag2, *ctx->next_Hi++);
+ ghash2_mul_first (&ctx->gd, aes_reflect (data) ^ tag2, *ctx->next_Hi++);
#else
#else
- ghash_mul_first (&ctx->gd, aes_
gcm_
reflect (data) ^ ctx->T, *ctx->next_Hi++);
+ ghash_mul_first (&ctx->gd, aes_reflect (data) ^ ctx->T, *ctx->next_Hi++);
#endif
}
static_always_inline void
aes_gcm_ghash_mul_next (aes_gcm_ctx_t *ctx, aes_data_t data)
{
#endif
}
static_always_inline void
aes_gcm_ghash_mul_next (aes_gcm_ctx_t *ctx, aes_data_t data)
{
-#if N_LANES == 4
- ghash4_mul_next (&ctx->gd, aes_
gcm_
reflect (data), *ctx->next_Hi++);
-#elif N_LANES == 2
- ghash2_mul_next (&ctx->gd, aes_
gcm_
reflect (data), *ctx->next_Hi++);
+#if N_
AES_
LANES == 4
+ ghash4_mul_next (&ctx->gd, aes_reflect (data), *ctx->next_Hi++);
+#elif N_
AES_
LANES == 2
+ ghash2_mul_next (&ctx->gd, aes_reflect (data), *ctx->next_Hi++);
#else
#else
- ghash_mul_next (&ctx->gd, aes_
gcm_
reflect (data), *ctx->next_Hi++);
+ ghash_mul_next (&ctx->gd, aes_reflect (data), *ctx->next_Hi++);
#endif
}
static_always_inline void
aes_gcm_ghash_mul_final_block (aes_gcm_ctx_t *ctx)
{
#endif
}
static_always_inline void
aes_gcm_ghash_mul_final_block (aes_gcm_ctx_t *ctx)
{
-#if N_LANES == 4
+#if N_
AES_
LANES == 4
u8x64 h = u8x64_insert_u8x16 (u8x64_zero (), ctx->Hi[NUM_HI - 1], 0);
u8x64 r4 = u8x64_insert_u8x16 (u8x64_zero (), aes_gcm_final_block (ctx), 0);
ghash4_mul_next (&ctx->gd, r4, h);
u8x64 h = u8x64_insert_u8x16 (u8x64_zero (), ctx->Hi[NUM_HI - 1], 0);
u8x64 r4 = u8x64_insert_u8x16 (u8x64_zero (), aes_gcm_final_block (ctx), 0);
ghash4_mul_next (&ctx->gd, r4, h);
-#elif N_LANES == 2
+#elif N_
AES_
LANES == 2
u8x32 h = u8x32_insert_lo (u8x32_zero (), ctx->Hi[NUM_HI - 1]);
u8x32 r2 = u8x32_insert_lo (u8x32_zero (), aes_gcm_final_block (ctx));
ghash2_mul_next (&ctx->gd, r2, h);
u8x32 h = u8x32_insert_lo (u8x32_zero (), ctx->Hi[NUM_HI - 1]);
u8x32 r2 = u8x32_insert_lo (u8x32_zero (), aes_gcm_final_block (ctx));
ghash2_mul_next (&ctx->gd, r2, h);
@@
-164,9
+133,9
@@
aes_gcm_enc_ctr0_round (aes_gcm_ctx_t *ctx, int aes_round)
if (aes_round == 0)
ctx->EY0 ^= ctx->Ke[0].x1;
else if (aes_round == ctx->rounds)
if (aes_round == 0)
ctx->EY0 ^= ctx->Ke[0].x1;
else if (aes_round == ctx->rounds)
- ctx->EY0 = aes_enc_last_round (ctx->EY0, ctx->Ke[aes_round].x1);
+ ctx->EY0 = aes_enc_last_round
_x1
(ctx->EY0, ctx->Ke[aes_round].x1);
else
else
- ctx->EY0 = aes_enc_round (ctx->EY0, ctx->Ke[aes_round].x1);
+ ctx->EY0 = aes_enc_round
_x1
(ctx->EY0, ctx->Ke[aes_round].x1);
}
static_always_inline void
}
static_always_inline void
@@
-176,11
+145,11
@@
aes_gcm_ghash (aes_gcm_ctx_t *ctx, u8 *data, u32 n_left)
aes_data_t r = {};
const aes_mem_t *d = (aes_mem_t *) data;
aes_data_t r = {};
const aes_mem_t *d = (aes_mem_t *) data;
- for (
; n_left >= 8 * N; n_left -= 8 * N
, d += 8)
+ for (
int n = 8 * N_AES_BYTES; n_left >= n; n_left -= n
, d += 8)
{
{
- if (ctx->operation == AES_GCM_OP_GMAC && n_left ==
N * 8
)
+ if (ctx->operation == AES_GCM_OP_GMAC && n_left ==
n
)
{
{
- aes_gcm_ghash_mul_first (ctx, d[0], 8 * N_LANES + 1);
+ aes_gcm_ghash_mul_first (ctx, d[0], 8 * N_
AES_
LANES + 1);
for (i = 1; i < 8; i++)
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_final_block (ctx);
for (i = 1; i < 8; i++)
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_final_block (ctx);
@@
-190,7
+159,7
@@
aes_gcm_ghash (aes_gcm_ctx_t *ctx, u8 *data, u32 n_left)
goto done;
}
goto done;
}
- aes_gcm_ghash_mul_first (ctx, d[0], 8 * N_LANES);
+ aes_gcm_ghash_mul_first (ctx, d[0], 8 * N_
AES_
LANES);
for (i = 1; i < 8; i++)
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_reduce (ctx);
for (i = 1; i < 8; i++)
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_reduce (ctx);
@@
-205,7
+174,7
@@
aes_gcm_ghash (aes_gcm_ctx_t *ctx, u8 *data, u32 n_left)
if (ctx->operation == AES_GCM_OP_GMAC)
n_lanes++;
if (ctx->operation == AES_GCM_OP_GMAC)
n_lanes++;
- if (n_left < N)
+ if (n_left < N
_AES_BYTES
)
{
clib_memcpy_fast (&r, d, n_left);
aes_gcm_ghash_mul_first (ctx, r, n_lanes);
{
clib_memcpy_fast (&r, d, n_left);
aes_gcm_ghash_mul_first (ctx, r, n_lanes);
@@
-213,30
+182,30
@@
aes_gcm_ghash (aes_gcm_ctx_t *ctx, u8 *data, u32 n_left)
else
{
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
else
{
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
- n_left -= N;
+ n_left -= N
_AES_BYTES
;
i = 1;
i = 1;
- if (n_left >= 4 * N)
+ if (n_left >= 4 * N
_AES_BYTES
)
{
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_next (ctx, d[i + 1]);
aes_gcm_ghash_mul_next (ctx, d[i + 2]);
aes_gcm_ghash_mul_next (ctx, d[i + 3]);
{
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_next (ctx, d[i + 1]);
aes_gcm_ghash_mul_next (ctx, d[i + 2]);
aes_gcm_ghash_mul_next (ctx, d[i + 3]);
- n_left -= 4 * N;
+ n_left -= 4 * N
_AES_BYTES
;
i += 4;
}
i += 4;
}
- if (n_left >= 2 * N)
+ if (n_left >= 2 * N
_AES_BYTES
)
{
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_next (ctx, d[i + 1]);
{
aes_gcm_ghash_mul_next (ctx, d[i]);
aes_gcm_ghash_mul_next (ctx, d[i + 1]);
- n_left -= 2 * N;
+ n_left -= 2 * N
_AES_BYTES
;
i += 2;
}
i += 2;
}
- if (n_left >= N)
+ if (n_left >= N
_AES_BYTES
)
{
aes_gcm_ghash_mul_next (ctx, d[i]);
{
aes_gcm_ghash_mul_next (ctx, d[i]);
- n_left -= N;
+ n_left -= N
_AES_BYTES
;
i += 1;
}
i += 1;
}
@@
-267,7
+236,7
@@
done:
static_always_inline void
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
{
static_always_inline void
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
{
- const aes_
gcm_
expaned_key_t Ke0 = ctx->Ke[0];
+ const aes_expaned_key_t Ke0 = ctx->Ke[0];
uword i = 0;
/* As counter is stored in network byte order for performance reasons we
uword i = 0;
/* As counter is stored in network byte order for performance reasons we
@@
-275,7
+244,7
@@
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
overlow. As we are processing four 128, 256 or 512-blocks in parallel
except the last round, overflow can happen only when n_blocks == 4 */
overlow. As we are processing four 128, 256 or 512-blocks in parallel
except the last round, overflow can happen only when n_blocks == 4 */
-#if N_LANES == 4
+#if N_
AES_
LANES == 4
const u32x16 ctr_inv_4444 = { 0, 0, 0, 4 << 24, 0, 0, 0, 4 << 24,
0, 0, 0, 4 << 24, 0, 0, 0, 4 << 24 };
const u32x16 ctr_inv_4444 = { 0, 0, 0, 4 << 24, 0, 0, 0, 4 << 24,
0, 0, 0, 4 << 24, 0, 0, 0, 4 << 24 };
@@
-292,13
+261,13
@@
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
if (n_blocks == 4 && PREDICT_FALSE ((u8) ctx->counter == 242))
{
if (n_blocks == 4 && PREDICT_FALSE ((u8) ctx->counter == 242))
{
- u32x16 Yr = (u32x16) aes_
gcm_
reflect ((u8x64) ctx->Y);
+ u32x16 Yr = (u32x16) aes_reflect ((u8x64) ctx->Y);
for (; i < n_blocks; i++)
{
r[i] = Ke0.x4 ^ (u8x64) ctx->Y; /* Initial AES round */
Yr += ctr_4444;
for (; i < n_blocks; i++)
{
r[i] = Ke0.x4 ^ (u8x64) ctx->Y; /* Initial AES round */
Yr += ctr_4444;
- ctx->Y = (u32x16) aes_
gcm_
reflect ((u8x64) Yr);
+ ctx->Y = (u32x16) aes_reflect ((u8x64) Yr);
}
}
else
}
}
else
@@
-310,7
+279,7
@@
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
}
}
ctx->counter += n_blocks * 4;
}
}
ctx->counter += n_blocks * 4;
-#elif N_LANES == 2
+#elif N_
AES_
LANES == 2
const u32x8 ctr_inv_22 = { 0, 0, 0, 2 << 24, 0, 0, 0, 2 << 24 };
const u32x8 ctr_22 = { 2, 0, 0, 0, 2, 0, 0, 0 };
const u32x8 ctr_inv_22 = { 0, 0, 0, 2 << 24, 0, 0, 0, 2 << 24 };
const u32x8 ctr_22 = { 2, 0, 0, 0, 2, 0, 0, 0 };
@@
-323,13
+292,13
@@
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
if (n_blocks == 4 && PREDICT_FALSE ((u8) ctx->counter == 250))
{
if (n_blocks == 4 && PREDICT_FALSE ((u8) ctx->counter == 250))
{
- u32x8 Yr = (u32x8) aes_
gcm_
reflect ((u8x32) ctx->Y);
+ u32x8 Yr = (u32x8) aes_reflect ((u8x32) ctx->Y);
for (; i < n_blocks; i++)
{
r[i] = Ke0.x2 ^ (u8x32) ctx->Y; /* Initial AES round */
Yr += ctr_22;
for (; i < n_blocks; i++)
{
r[i] = Ke0.x2 ^ (u8x32) ctx->Y; /* Initial AES round */
Yr += ctr_22;
- ctx->Y = (u32x8) aes_
gcm_
reflect ((u8x32) Yr);
+ ctx->Y = (u32x8) aes_reflect ((u8x32) Yr);
}
}
else
}
}
else
@@
-369,43
+338,22
@@
aes_gcm_enc_first_round (aes_gcm_ctx_t *ctx, aes_data_t *r, uword n_blocks)
#endif
}
#endif
}
-static_always_inline void
-aes_gcm_enc_round (aes_data_t *r, const aes_gcm_expaned_key_t *Ke,
- uword n_blocks)
-{
- for (int i = 0; i < n_blocks; i++)
-#if N_LANES == 4
- r[i] = aes_enc_round_x4 (r[i], Ke->x4);
-#elif N_LANES == 2
- r[i] = aes_enc_round_x2 (r[i], Ke->x2);
-#else
- r[i] = aes_enc_round (r[i], Ke->x1);
-#endif
-}
-
static_always_inline void
aes_gcm_enc_last_round (aes_gcm_ctx_t *ctx, aes_data_t *r, aes_data_t *d,
static_always_inline void
aes_gcm_enc_last_round (aes_gcm_ctx_t *ctx, aes_data_t *r, aes_data_t *d,
- const aes_
gcm_
expaned_key_t *Ke, uword n_blocks)
+ const aes_expaned_key_t *Ke, uword n_blocks)
{
/* additional ronuds for AES-192 and AES-256 */
for (int i = 10; i < ctx->rounds; i++)
{
/* additional ronuds for AES-192 and AES-256 */
for (int i = 10; i < ctx->rounds; i++)
- aes_
gcm_
enc_round (r, Ke + i, n_blocks);
+ aes_enc_round (r, Ke + i, n_blocks);
- for (int i = 0; i < n_blocks; i++)
-#if N_LANES == 4
- d[i] ^= aes_enc_last_round_x4 (r[i], Ke[ctx->rounds].x4);
-#elif N_LANES == 2
- d[i] ^= aes_enc_last_round_x2 (r[i], Ke[ctx->rounds].x2);
-#else
- d[i] ^= aes_enc_last_round (r[i], Ke[ctx->rounds].x1);
-#endif
+ aes_enc_last_round (r, d, Ke + ctx->rounds, n_blocks);
}
static_always_inline void
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
u32 n_bytes, int with_ghash)
{
}
static_always_inline void
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
u32 n_bytes, int with_ghash)
{
- const aes_
gcm_
expaned_key_t *k = ctx->Ke;
+ const aes_expaned_key_t *k = ctx->Ke;
const aes_mem_t *sv = (aes_mem_t *) src;
aes_mem_t *dv = (aes_mem_t *) dst;
uword ghash_blocks, gc = 1;
const aes_mem_t *sv = (aes_mem_t *) src;
aes_mem_t *dv = (aes_mem_t *) dst;
uword ghash_blocks, gc = 1;
@@
-415,23
+363,23
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
if (ctx->operation == AES_GCM_OP_ENCRYPT)
{
ghash_blocks = 4;
if (ctx->operation == AES_GCM_OP_ENCRYPT)
{
ghash_blocks = 4;
- n_lanes = N_LANES * 4;
+ n_lanes = N_
AES_
LANES * 4;
}
else
{
ghash_blocks = n;
}
else
{
ghash_blocks = n;
- n_lanes = n * N_LANES;
-#if N_LANES != 1
+ n_lanes = n * N_
AES_
LANES;
+#if N_
AES_
LANES != 1
if (ctx->last)
n_lanes = (n_bytes + 15) / 16;
#endif
}
if (ctx->last)
n_lanes = (n_bytes + 15) / 16;
#endif
}
- n_bytes -= (n - 1) * N;
+ n_bytes -= (n - 1) * N
_AES_BYTES
;
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, n);
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, n);
- aes_
gcm_
enc_round (r, k + 1, n);
+ aes_enc_round (r, k + 1, n);
/* load data - decrypt round */
if (ctx->operation == AES_GCM_OP_DECRYPT)
/* load data - decrypt round */
if (ctx->operation == AES_GCM_OP_DECRYPT)
@@
-440,7
+388,7
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
d[i] = sv[i];
if (ctx->last)
d[i] = sv[i];
if (ctx->last)
- d[n - 1] = aes_
gcm_
load_partial ((u8 *) (sv + n - 1), n_bytes);
+ d[n - 1] = aes_load_partial ((u8 *) (sv + n - 1), n_bytes);
}
/* GHASH multiply block 0 */
}
/* GHASH multiply block 0 */
@@
-448,24
+396,24
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
/* AES rounds 2 and 3 */
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
/* AES rounds 2 and 3 */
- aes_
gcm_
enc_round (r, k + 2, n);
- aes_
gcm_
enc_round (r, k + 3, n);
+ aes_enc_round (r, k + 2, n);
+ aes_enc_round (r, k + 3, n);
/* GHASH multiply block 1 */
if (with_ghash && gc++ < ghash_blocks)
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 4 and 5 */
/* GHASH multiply block 1 */
if (with_ghash && gc++ < ghash_blocks)
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 4 and 5 */
- aes_
gcm_
enc_round (r, k + 4, n);
- aes_
gcm_
enc_round (r, k + 5, n);
+ aes_enc_round (r, k + 4, n);
+ aes_enc_round (r, k + 5, n);
/* GHASH multiply block 2 */
if (with_ghash && gc++ < ghash_blocks)
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 6 and 7 */
/* GHASH multiply block 2 */
if (with_ghash && gc++ < ghash_blocks)
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 6 and 7 */
- aes_
gcm_
enc_round (r, k + 6, n);
- aes_
gcm_
enc_round (r, k + 7, n);
+ aes_enc_round (r, k + 6, n);
+ aes_enc_round (r, k + 7, n);
/* GHASH multiply block 3 */
if (with_ghash && gc++ < ghash_blocks)
/* GHASH multiply block 3 */
if (with_ghash && gc++ < ghash_blocks)
@@
-478,12
+426,12
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
d[i] = sv[i];
if (ctx->last)
d[i] = sv[i];
if (ctx->last)
- d[n - 1] = aes_
gcm_
load_partial (sv + n - 1, n_bytes);
+ d[n - 1] = aes_load_partial (sv + n - 1, n_bytes);
}
/* AES rounds 8 and 9 */
}
/* AES rounds 8 and 9 */
- aes_
gcm_
enc_round (r, k + 8, n);
- aes_
gcm_
enc_round (r, k + 9, n);
+ aes_enc_round (r, k + 8, n);
+ aes_enc_round (r, k + 9, n);
/* AES last round(s) */
aes_gcm_enc_last_round (ctx, r, d, k, n);
/* AES last round(s) */
aes_gcm_enc_last_round (ctx, r, d, k, n);
@@
-493,7
+441,7
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
dv[i] = d[i];
if (ctx->last)
dv[i] = d[i];
if (ctx->last)
- aes_
gcm_
store_partial (d[n - 1], dv + n - 1, n_bytes);
+ aes_store_partial (d[n - 1], dv + n - 1, n_bytes);
/* GHASH reduce 1st step */
aes_gcm_ghash_reduce (ctx);
/* GHASH reduce 1st step */
aes_gcm_ghash_reduce (ctx);
@@
-510,14
+458,14
@@
aes_gcm_calc (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst, u32 n,
static_always_inline void
aes_gcm_calc_double (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst)
{
static_always_inline void
aes_gcm_calc_double (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst)
{
- const aes_
gcm_
expaned_key_t *k = ctx->Ke;
+ const aes_expaned_key_t *k = ctx->Ke;
const aes_mem_t *sv = (aes_mem_t *) src;
aes_mem_t *dv = (aes_mem_t *) dst;
aes_data_t r[4];
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, 4);
const aes_mem_t *sv = (aes_mem_t *) src;
aes_mem_t *dv = (aes_mem_t *) dst;
aes_data_t r[4];
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, 4);
- aes_
gcm_
enc_round (r, k + 1, 4);
+ aes_enc_round (r, k + 1, 4);
/* load 4 blocks of data - decrypt round */
if (ctx->operation == AES_GCM_OP_DECRYPT)
/* load 4 blocks of data - decrypt round */
if (ctx->operation == AES_GCM_OP_DECRYPT)
@@
-525,32
+473,32
@@
aes_gcm_calc_double (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst)
d[i] = sv[i];
/* GHASH multiply block 0 */
d[i] = sv[i];
/* GHASH multiply block 0 */
- aes_gcm_ghash_mul_first (ctx, d[0], N_LANES * 8);
+ aes_gcm_ghash_mul_first (ctx, d[0], N_
AES_
LANES * 8);
/* AES rounds 2 and 3 */
/* AES rounds 2 and 3 */
- aes_
gcm_
enc_round (r, k + 2, 4);
- aes_
gcm_
enc_round (r, k + 3, 4);
+ aes_enc_round (r, k + 2, 4);
+ aes_enc_round (r, k + 3, 4);
/* GHASH multiply block 1 */
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 4 and 5 */
/* GHASH multiply block 1 */
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 4 and 5 */
- aes_
gcm_
enc_round (r, k + 4, 4);
- aes_
gcm_
enc_round (r, k + 5, 4);
+ aes_enc_round (r, k + 4, 4);
+ aes_enc_round (r, k + 5, 4);
/* GHASH multiply block 2 */
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 6 and 7 */
/* GHASH multiply block 2 */
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 6 and 7 */
- aes_
gcm_
enc_round (r, k + 6, 4);
- aes_
gcm_
enc_round (r, k + 7, 4);
+ aes_enc_round (r, k + 6, 4);
+ aes_enc_round (r, k + 7, 4);
/* GHASH multiply block 3 */
aes_gcm_ghash_mul_next (ctx, (d[3]));
/* AES rounds 8 and 9 */
/* GHASH multiply block 3 */
aes_gcm_ghash_mul_next (ctx, (d[3]));
/* AES rounds 8 and 9 */
- aes_
gcm_
enc_round (r, k + 8, 4);
- aes_
gcm_
enc_round (r, k + 9, 4);
+ aes_enc_round (r, k + 8, 4);
+ aes_enc_round (r, k + 9, 4);
/* load 4 blocks of data - encrypt round */
if (ctx->operation == AES_GCM_OP_ENCRYPT)
/* load 4 blocks of data - encrypt round */
if (ctx->operation == AES_GCM_OP_ENCRYPT)
@@
-574,35
+522,35
@@
aes_gcm_calc_double (aes_gcm_ctx_t *ctx, aes_data_t *d, const u8 *src, u8 *dst)
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, 4);
/* AES rounds 0 and 1 */
aes_gcm_enc_first_round (ctx, r, 4);
- aes_
gcm_
enc_round (r, k + 1, 4);
+ aes_enc_round (r, k + 1, 4);
/* GHASH multiply block 5 */
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 2 and 3 */
/* GHASH multiply block 5 */
aes_gcm_ghash_mul_next (ctx, (d[1]));
/* AES rounds 2 and 3 */
- aes_
gcm_
enc_round (r, k + 2, 4);
- aes_
gcm_
enc_round (r, k + 3, 4);
+ aes_enc_round (r, k + 2, 4);
+ aes_enc_round (r, k + 3, 4);
/* GHASH multiply block 6 */
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 4 and 5 */
/* GHASH multiply block 6 */
aes_gcm_ghash_mul_next (ctx, (d[2]));
/* AES rounds 4 and 5 */
- aes_
gcm_
enc_round (r, k + 4, 4);
- aes_
gcm_
enc_round (r, k + 5, 4);
+ aes_enc_round (r, k + 4, 4);
+ aes_enc_round (r, k + 5, 4);
/* GHASH multiply block 7 */
aes_gcm_ghash_mul_next (ctx, (d[3]));
/* AES rounds 6 and 7 */
/* GHASH multiply block 7 */
aes_gcm_ghash_mul_next (ctx, (d[3]));
/* AES rounds 6 and 7 */
- aes_
gcm_
enc_round (r, k + 6, 4);
- aes_
gcm_
enc_round (r, k + 7, 4);
+ aes_enc_round (r, k + 6, 4);
+ aes_enc_round (r, k + 7, 4);
/* GHASH reduce 1st step */
aes_gcm_ghash_reduce (ctx);
/* AES rounds 8 and 9 */
/* GHASH reduce 1st step */
aes_gcm_ghash_reduce (ctx);
/* AES rounds 8 and 9 */
- aes_
gcm_
enc_round (r, k + 8, 4);
- aes_
gcm_
enc_round (r, k + 9, 4);
+ aes_enc_round (r, k + 8, 4);
+ aes_enc_round (r, k + 9, 4);
/* GHASH reduce 2nd step */
aes_gcm_ghash_reduce2 (ctx);
/* GHASH reduce 2nd step */
aes_gcm_ghash_reduce2 (ctx);
@@
-644,14
+592,14
@@
static_always_inline void
aes_gcm_calc_last (aes_gcm_ctx_t *ctx, aes_data_t *d, int n_blocks,
u32 n_bytes)
{
aes_gcm_calc_last (aes_gcm_ctx_t *ctx, aes_data_t *d, int n_blocks,
u32 n_bytes)
{
- int n_lanes = (N_LANES == 1 ? n_blocks : (n_bytes + 15) / 16) + 1;
- n_bytes -= (n_blocks - 1) * N;
+ int n_lanes = (N_
AES_
LANES == 1 ? n_blocks : (n_bytes + 15) / 16) + 1;
+ n_bytes -= (n_blocks - 1) * N
_AES_BYTES
;
int i;
aes_gcm_enc_ctr0_round (ctx, 0);
aes_gcm_enc_ctr0_round (ctx, 1);
int i;
aes_gcm_enc_ctr0_round (ctx, 0);
aes_gcm_enc_ctr0_round (ctx, 1);
- if (n_bytes != N)
+ if (n_bytes != N
_AES_BYTES
)
aes_gcm_mask_bytes (d + n_blocks - 1, n_bytes);
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
aes_gcm_mask_bytes (d + n_blocks - 1, n_bytes);
aes_gcm_ghash_mul_first (ctx, d[0], n_lanes);
@@
-703,20
+651,20
@@
aes_gcm_enc (aes_gcm_ctx_t *ctx, const u8 *src, u8 *dst, u32 n_left)
return;
}
return;
}
- if (n_left < 4 * N)
+ if (n_left < 4 * N
_AES_BYTES
)
{
ctx->last = 1;
{
ctx->last = 1;
- if (n_left > 3 * N)
+ if (n_left > 3 * N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 4, n_left);
}
{
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 4, n_left);
}
- else if (n_left > 2 * N)
+ else if (n_left > 2 * N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 3, n_left);
}
{
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 3, n_left);
}
- else if (n_left > N)
+ else if (n_left > N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 2, n_left);
{
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 0);
aes_gcm_calc_last (ctx, d, 2, n_left);
@@
-729,45
+677,45
@@
aes_gcm_enc (aes_gcm_ctx_t *ctx, const u8 *src, u8 *dst, u32 n_left)
return;
}
return;
}
- aes_gcm_calc (ctx, d, src, dst, 4, 4 * N, /* with_ghash */ 0);
+ aes_gcm_calc (ctx, d, src, dst, 4, 4 * N
_AES_BYTES
, /* with_ghash */ 0);
/* next */
/* next */
- n_left -= 4 * N;
- dst += 4 * N;
- src += 4 * N;
+ n_left -= 4 * N
_AES_BYTES
;
+ dst += 4 * N
_AES_BYTES
;
+ src += 4 * N
_AES_BYTES
;
- for (
; n_left >= 8 * N; n_left -= 8 * N, src += 8 * N, dst += 8 * N
)
+ for (
int n = 8 * N_AES_BYTES; n_left >= n; n_left -= n, src += n, dst += n
)
aes_gcm_calc_double (ctx, d, src, dst);
aes_gcm_calc_double (ctx, d, src, dst);
- if (n_left >= 4 * N)
+ if (n_left >= 4 * N
_AES_BYTES
)
{
{
- aes_gcm_calc (ctx, d, src, dst, 4, 4 * N, /* with_ghash */ 1);
+ aes_gcm_calc (ctx, d, src, dst, 4, 4 * N
_AES_BYTES
, /* with_ghash */ 1);
/* next */
/* next */
- n_left -= 4 * N;
- dst += 4 * N;
- src += 4 * N;
+ n_left -= 4 * N
_AES_BYTES
;
+ dst += 4 * N
_AES_BYTES
;
+ src += 4 * N
_AES_BYTES
;
}
if (n_left == 0)
{
}
if (n_left == 0)
{
- aes_gcm_calc_last (ctx, d, 4, 4 * N);
+ aes_gcm_calc_last (ctx, d, 4, 4 * N
_AES_BYTES
);
return;
}
ctx->last = 1;
return;
}
ctx->last = 1;
- if (n_left > 3 * N)
+ if (n_left > 3 * N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 4, n_left);
}
{
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 4, n_left);
}
- else if (n_left > 2 * N)
+ else if (n_left > 2 * N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 3, n_left);
}
{
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 3, n_left);
}
- else if (n_left > N)
+ else if (n_left > N
_AES_BYTES
)
{
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 2, n_left);
{
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 1);
aes_gcm_calc_last (ctx, d, 2, n_left);
@@
-786,28
+734,28
@@
aes_gcm_dec (aes_gcm_ctx_t *ctx, const u8 *src, u8 *dst, uword n_left)
ghash_ctx_t gd;
/* main encryption loop */
ghash_ctx_t gd;
/* main encryption loop */
- for (
; n_left >= 8 * N; n_left -= 8 * N, dst += 8 * N, src += 8 * N
)
+ for (
int n = 8 * N_AES_BYTES; n_left >= n; n_left -= n, dst += n, src += n
)
aes_gcm_calc_double (ctx, d, src, dst);
aes_gcm_calc_double (ctx, d, src, dst);
- if (n_left >= 4 * N)
+ if (n_left >= 4 * N
_AES_BYTES
)
{
{
- aes_gcm_calc (ctx, d, src, dst, 4, 4 * N, /* with_ghash */ 1);
+ aes_gcm_calc (ctx, d, src, dst, 4, 4 * N
_AES_BYTES
, /* with_ghash */ 1);
/* next */
/* next */
- n_left -= 4 * N;
- dst += N * 4;
- src += N * 4;
+ n_left -= 4 * N
_AES_BYTES
;
+ dst += N
_AES_BYTES
* 4;
+ src += N
_AES_BYTES
* 4;
}
if (n_left)
{
ctx->last = 1;
}
if (n_left)
{
ctx->last = 1;
- if (n_left > 3 * N)
+ if (n_left > 3 * N
_AES_BYTES
)
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 1);
aes_gcm_calc (ctx, d, src, dst, 4, n_left, /* with_ghash */ 1);
- else if (n_left > 2 * N)
+ else if (n_left > 2 * N
_AES_BYTES
)
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 1);
aes_gcm_calc (ctx, d, src, dst, 3, n_left, /* with_ghash */ 1);
- else if (n_left > N)
+ else if (n_left > N
_AES_BYTES
)
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 1);
else
aes_gcm_calc (ctx, d, src, dst, 1, n_left, /* with_ghash */ 1);
aes_gcm_calc (ctx, d, src, dst, 2, n_left, /* with_ghash */ 1);
else
aes_gcm_calc (ctx, d, src, dst, 1, n_left, /* with_ghash */ 1);
@@
-867,11
+815,11
@@
aes_gcm (const u8 *src, u8 *dst, const u8 *aad, u8 *ivp, u8 *tag,
Y0[3] = 1 << 24;
ctx->EY0 = (u8x16) Y0;
Y0[3] = 1 << 24;
ctx->EY0 = (u8x16) Y0;
-#if N_LANES == 4
+#if N_
AES_
LANES == 4
ctx->Y = u32x16_splat_u32x4 (Y0) + (u32x16){
0, 0, 0, 1 << 24, 0, 0, 0, 2 << 24, 0, 0, 0, 3 << 24, 0, 0, 0, 4 << 24,
};
ctx->Y = u32x16_splat_u32x4 (Y0) + (u32x16){
0, 0, 0, 1 << 24, 0, 0, 0, 2 << 24, 0, 0, 0, 3 << 24, 0, 0, 0, 4 << 24,
};
-#elif N_LANES == 2
+#elif N_
AES_
LANES == 2
ctx->Y =
u32x8_splat_u32x4 (Y0) + (u32x8){ 0, 0, 0, 1 << 24, 0, 0, 0, 2 << 24 };
#else
ctx->Y =
u32x8_splat_u32x4 (Y0) + (u32x8){ 0, 0, 0, 1 << 24, 0, 0, 0, 2 << 24 };
#else
@@
-926,7
+874,7
@@
clib_aes_gcm_key_expand (aes_gcm_key_data_t *kd, const u8 *key,
{
u8x16 H;
u8x16 ek[AES_KEY_ROUNDS (AES_KEY_256) + 1];
{
u8x16 H;
u8x16 ek[AES_KEY_ROUNDS (AES_KEY_256) + 1];
- aes_
gcm_expaned_key_t *Ke = (aes_gcm
_expaned_key_t *) kd->Ke;
+ aes_
expaned_key_t *Ke = (aes
_expaned_key_t *) kd->Ke;
/* expand AES key */
aes_key_expand (ek, key, ks);
/* expand AES key */
aes_key_expand (ek, key, ks);