drivers/crypto/aesni_mb/aesni_mb_ops.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015 Intel Corporation
   3  */
   4
   5 #ifndef _AESNI_MB_OPS_H_
   6 #define _AESNI_MB_OPS_H_
   7
   8 #ifndef LINUX
   9 #define LINUX
  10 #endif
  11
  12 #include <intel-ipsec-mb.h>
  13
  14 /*
  15  * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
  16  * so if macro is not defined, it means that the version is 0.49.
  17  */
  18 #if !defined(IMB_VERSION_NUM)
  19 #define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
  20 #define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
  21 #endif
  22
  23 enum aesni_mb_vector_mode {
  24         RTE_AESNI_MB_NOT_SUPPORTED = 0,
  25         RTE_AESNI_MB_SSE,
  26         RTE_AESNI_MB_AVX,
  27         RTE_AESNI_MB_AVX2,
  28         RTE_AESNI_MB_AVX512
  29 };
  30
  31 typedef void (*md5_one_block_t)(const void *data, void *digest);
  32
  33 typedef void (*sha1_one_block_t)(const void *data, void *digest);
  34 typedef void (*sha224_one_block_t)(const void *data, void *digest);
  35 typedef void (*sha256_one_block_t)(const void *data, void *digest);
  36 typedef void (*sha384_one_block_t)(const void *data, void *digest);
  37 typedef void (*sha512_one_block_t)(const void *data, void *digest);
  38
  39 typedef void (*aes_keyexp_128_t)
  40                 (const void *key, void *enc_exp_keys, void *dec_exp_keys);
  41 typedef void (*aes_keyexp_192_t)
  42                 (const void *key, void *enc_exp_keys, void *dec_exp_keys);
  43 typedef void (*aes_keyexp_256_t)
  44                 (const void *key, void *enc_exp_keys, void *dec_exp_keys);
  45 typedef void (*aes_xcbc_expand_key_t)
  46                 (const void *key, void *exp_k1, void *k2, void *k3);
  47 typedef void (*aes_cmac_sub_key_gen_t)
  48                 (const void *exp_key, void *k2, void *k3);
  49 typedef void (*aes_cmac_keyexp_t)
  50                 (const void *key, void *keyexp);
  51 typedef void (*aes_gcm_keyexp_t)
  52                 (const void *key, struct gcm_key_data *keyexp);
  53
  54 /** Multi-buffer library function pointer table */
  55 struct aesni_mb_op_fns {
  56         struct {
  57                 init_mb_mgr_t init_mgr;
  58                 /**< Initialise scheduler  */
  59                 get_next_job_t get_next;
  60                 /**< Get next free job structure */
  61                 submit_job_t submit;
  62                 /**< Submit job to scheduler */
  63                 get_completed_job_t get_completed_job;
  64                 /**< Get completed job */
  65                 flush_job_t flush_job;
  66                 /**< flush jobs from manager */
  67         } job;
  68         /**< multi buffer manager functions */
  69
  70         struct {
  71                 struct {
  72                         md5_one_block_t md5;
  73                         /**< MD5 one block hash */
  74                         sha1_one_block_t sha1;
  75                         /**< SHA1 one block hash */
  76                         sha224_one_block_t sha224;
  77                         /**< SHA224 one block hash */
  78                         sha256_one_block_t sha256;
  79                         /**< SHA256 one block hash */
  80                         sha384_one_block_t sha384;
  81                         /**< SHA384 one block hash */
  82                         sha512_one_block_t sha512;
  83                         /**< SHA512 one block hash */
  84                 } one_block;
  85                 /**< one block hash functions */
  86
  87                 struct {
  88                         aes_keyexp_128_t aes128;
  89                         /**< AES128 key expansions */
  90                         aes_keyexp_192_t aes192;
  91                         /**< AES192 key expansions */
  92                         aes_keyexp_256_t aes256;
  93                         /**< AES256 key expansions */
  94                         aes_xcbc_expand_key_t aes_xcbc;
  95                         /**< AES XCBC key epansions */
  96                         aes_cmac_sub_key_gen_t aes_cmac_subkey;
  97                         /**< AES CMAC subkey expansions */
  98                         aes_cmac_keyexp_t aes_cmac_expkey;
  99                         /**< AES CMAC key expansions */
 100                         aes_gcm_keyexp_t aes_gcm_128;
 101                         /**< AES GCM 128 key expansions */
 102                         aes_gcm_keyexp_t aes_gcm_192;
 103                         /**< AES GCM 192 key expansions */
 104                         aes_gcm_keyexp_t aes_gcm_256;
 105                         /**< AES GCM 256 key expansions */
 106                 } keyexp;
 107                 /**< Key expansion functions */
 108 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 109                 struct {
 110                         hash_fn_t sha1;
 111                         hash_fn_t sha224;
 112                         hash_fn_t sha256;
 113                         hash_fn_t sha384;
 114                         hash_fn_t sha512;
 115                 } multi_block;
 116                 /** multi block hash functions */
 117 #endif
 118         } aux;
 119         /**< Auxiliary functions */
 120 };
 121
 122
 123 static const struct aesni_mb_op_fns job_ops[] = {
 124                 [RTE_AESNI_MB_NOT_SUPPORTED] = {
 125                         .job = {
 126                                 NULL
 127                         },
 128                         .aux = {
 129                                 .one_block = {
 130                                         NULL
 131                                 },
 132                                 .keyexp = {
 133                                         NULL
 134                                 },
 135 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 136                                 .multi_block = {
 137                                         NULL
 138                                 }
 139 #endif
 140
 141                         }
 142                 },
 143                 [RTE_AESNI_MB_SSE] = {
 144                         .job = {
 145                                 init_mb_mgr_sse,
 146                                 get_next_job_sse,
 147                                 submit_job_sse,
 148                                 get_completed_job_sse,
 149                                 flush_job_sse
 150                         },
 151                         .aux = {
 152                                 .one_block = {
 153                                         md5_one_block_sse,
 154                                         sha1_one_block_sse,
 155                                         sha224_one_block_sse,
 156                                         sha256_one_block_sse,
 157                                         sha384_one_block_sse,
 158                                         sha512_one_block_sse
 159                                 },
 160                                 .keyexp = {
 161                                         aes_keyexp_128_sse,
 162                                         aes_keyexp_192_sse,
 163                                         aes_keyexp_256_sse,
 164                                         aes_xcbc_expand_key_sse,
 165                                         aes_cmac_subkey_gen_sse,
 166                                         aes_keyexp_128_enc_sse,
 167                                         aes_gcm_pre_128_sse,
 168                                         aes_gcm_pre_192_sse,
 169                                         aes_gcm_pre_256_sse
 170                                 },
 171 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 172                                 .multi_block = {
 173                                         sha1_sse,
 174                                         sha224_sse,
 175                                         sha256_sse,
 176                                         sha384_sse,
 177                                         sha512_sse
 178                                 }
 179 #endif
 180                         }
 181                 },
 182                 [RTE_AESNI_MB_AVX] = {
 183                         .job = {
 184                                 init_mb_mgr_avx,
 185                                 get_next_job_avx,
 186                                 submit_job_avx,
 187                                 get_completed_job_avx,
 188                                 flush_job_avx
 189                         },
 190                         .aux = {
 191                                 .one_block = {
 192                                         md5_one_block_avx,
 193                                         sha1_one_block_avx,
 194                                         sha224_one_block_avx,
 195                                         sha256_one_block_avx,
 196                                         sha384_one_block_avx,
 197                                         sha512_one_block_avx
 198                                 },
 199                                 .keyexp = {
 200                                         aes_keyexp_128_avx,
 201                                         aes_keyexp_192_avx,
 202                                         aes_keyexp_256_avx,
 203                                         aes_xcbc_expand_key_avx,
 204                                         aes_cmac_subkey_gen_avx,
 205                                         aes_keyexp_128_enc_avx,
 206                                         aes_gcm_pre_128_avx_gen2,
 207                                         aes_gcm_pre_192_avx_gen2,
 208                                         aes_gcm_pre_256_avx_gen2
 209                                 },
 210 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 211                                 .multi_block = {
 212                                         sha1_avx,
 213                                         sha224_avx,
 214                                         sha256_avx,
 215                                         sha384_avx,
 216                                         sha512_avx
 217                                 }
 218 #endif
 219                         }
 220                 },
 221                 [RTE_AESNI_MB_AVX2] = {
 222                         .job = {
 223                                 init_mb_mgr_avx2,
 224                                 get_next_job_avx2,
 225                                 submit_job_avx2,
 226                                 get_completed_job_avx2,
 227                                 flush_job_avx2
 228                         },
 229                         .aux = {
 230                                 .one_block = {
 231                                         md5_one_block_avx2,
 232                                         sha1_one_block_avx2,
 233                                         sha224_one_block_avx2,
 234                                         sha256_one_block_avx2,
 235                                         sha384_one_block_avx2,
 236                                         sha512_one_block_avx2
 237                                 },
 238                                 .keyexp = {
 239                                         aes_keyexp_128_avx2,
 240                                         aes_keyexp_192_avx2,
 241                                         aes_keyexp_256_avx2,
 242                                         aes_xcbc_expand_key_avx2,
 243                                         aes_cmac_subkey_gen_avx2,
 244                                         aes_keyexp_128_enc_avx2,
 245                                         aes_gcm_pre_128_avx_gen4,
 246                                         aes_gcm_pre_192_avx_gen4,
 247                                         aes_gcm_pre_256_avx_gen4
 248                                 },
 249 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 250                                 .multi_block = {
 251                                         sha1_avx2,
 252                                         sha224_avx2,
 253                                         sha256_avx2,
 254                                         sha384_avx2,
 255                                         sha512_avx2
 256                                 }
 257 #endif
 258                         }
 259                 },
 260                 [RTE_AESNI_MB_AVX512] = {
 261                         .job = {
 262                                 init_mb_mgr_avx512,
 263                                 get_next_job_avx512,
 264                                 submit_job_avx512,
 265                                 get_completed_job_avx512,
 266                                 flush_job_avx512
 267                         },
 268                         .aux = {
 269                                 .one_block = {
 270                                         md5_one_block_avx512,
 271                                         sha1_one_block_avx512,
 272                                         sha224_one_block_avx512,
 273                                         sha256_one_block_avx512,
 274                                         sha384_one_block_avx512,
 275                                         sha512_one_block_avx512
 276                                 },
 277                                 .keyexp = {
 278                                         aes_keyexp_128_avx512,
 279                                         aes_keyexp_192_avx512,
 280                                         aes_keyexp_256_avx512,
 281                                         aes_xcbc_expand_key_avx512,
 282                                         aes_cmac_subkey_gen_avx512,
 283                                         aes_keyexp_128_enc_avx512,
 284                                         aes_gcm_pre_128_avx_gen4,
 285                                         aes_gcm_pre_192_avx_gen4,
 286                                         aes_gcm_pre_256_avx_gen4
 287                                 },
 288 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 289                                 .multi_block = {
 290                                         sha1_avx512,
 291                                         sha224_avx512,
 292                                         sha256_avx512,
 293                                         sha384_avx512,
 294                                         sha512_avx512
 295                                 }
 296 #endif
 297                         }
 298                 }
 299 };
 300
 301
 302 #endif /* _AESNI_MB_OPS_H_ */