src/vnet/crypto/crypto.h

   1 /*
   2  * Copyright (c) 2019 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #ifndef included_vnet_crypto_crypto_h
  17 #define included_vnet_crypto_crypto_h
  18
  19 #include <vlib/vlib.h>
  20
  21 #define VNET_CRYPTO_FRAME_SIZE 64
  22
  23 /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES */
  24 #define foreach_crypto_cipher_alg \
  25   _(DES_CBC,     "des-cbc", 7) \
  26   _(3DES_CBC,    "3des-cbc", 24) \
  27   _(AES_128_CBC, "aes-128-cbc", 16) \
  28   _(AES_192_CBC, "aes-192-cbc", 24) \
  29   _(AES_256_CBC, "aes-256-cbc", 32) \
  30   _(AES_128_CTR, "aes-128-ctr", 16) \
  31   _(AES_192_CTR, "aes-192-ctr", 24) \
  32   _(AES_256_CTR, "aes-256-ctr", 32)
  33
  34 /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES */
  35 #define foreach_crypto_aead_alg \
  36   _(AES_128_GCM, "aes-128-gcm", 16) \
  37   _(AES_192_GCM, "aes-192-gcm", 24) \
  38   _(AES_256_GCM, "aes-256-gcm", 32) \
  39   _(CHACHA20_POLY1305, "chacha20-poly1305", 32)
  40
  41 #define foreach_crypto_hmac_alg \
  42   _(MD5, "md5") \
  43   _(SHA1, "sha-1") \
  44   _(SHA224, "sha-224")  \
  45   _(SHA256, "sha-256")  \
  46   _(SHA384, "sha-384")  \
  47   _(SHA512, "sha-512")
  48
  49 #define foreach_crypto_op_type \
  50   _(ENCRYPT, "encrypt") \
  51   _(DECRYPT, "decrypt") \
  52   _(AEAD_ENCRYPT, "aead-encrypt") \
  53   _(AEAD_DECRYPT, "aead-decrypt") \
  54   _(HMAC, "hmac")
  55
  56 typedef enum
  57 {
  58 #define _(n, s) VNET_CRYPTO_OP_TYPE_##n,
  59   foreach_crypto_op_type
  60 #undef _
  61     VNET_CRYPTO_OP_N_TYPES,
  62 } vnet_crypto_op_type_t;
  63
  64 #define foreach_crypto_op_status \
  65   _(IDLE, "idle") \
  66   _(PENDING, "pending") \
  67   _(WORK_IN_PROGRESS, "work-in-progress") \
  68   _(COMPLETED, "completed") \
  69   _(FAIL_NO_HANDLER, "no-handler") \
  70   _(FAIL_BAD_HMAC, "bad-hmac") \
  71   _(FAIL_ENGINE_ERR, "engine-error")
  72
  73 /** async crypto **/
  74
  75 /* CRYPTO_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, TAG_LEN, AAD_LEN */
  76 #define foreach_crypto_aead_async_alg \
  77   _(AES_128_GCM, "aes-128-gcm-aad8", 16, 16, 8) \
  78   _(AES_128_GCM, "aes-128-gcm-aad12", 16, 16, 12) \
  79   _(AES_192_GCM, "aes-192-gcm-aad8", 24, 16, 8) \
  80   _(AES_192_GCM, "aes-192-gcm-aad12", 24, 16, 12) \
  81   _(AES_256_GCM, "aes-256-gcm-aad8", 32, 16, 8) \
  82   _(AES_256_GCM, "aes-256-gcm-aad12", 32, 16, 12) \
  83   _(CHACHA20_POLY1305, "chacha20-poly1305-aad8", 32, 16, 8) \
  84   _(CHACHA20_POLY1305, "chacha20-poly1305-aad12", 32, 16, 12)
  85
  86 /* CRYPTO_ID, INTEG_ID, PRETTY_NAME, KEY_LENGTH_IN_BYTES, DIGEST_LEN */
  87 #define foreach_crypto_link_async_alg \
  88   _ (AES_128_CBC, SHA1, "aes-128-cbc-hmac-sha-1", 16, 12) \
  89   _ (AES_192_CBC, SHA1, "aes-192-cbc-hmac-sha-1", 24, 12) \
  90   _ (AES_256_CBC, SHA1, "aes-256-cbc-hmac-sha-1", 32, 12) \
  91   _ (AES_128_CBC, SHA224, "aes-128-cbc-hmac-sha-224", 16, 14) \
  92   _ (AES_192_CBC, SHA224, "aes-192-cbc-hmac-sha-224", 24, 14) \
  93   _ (AES_256_CBC, SHA224, "aes-256-cbc-hmac-sha-224", 32, 14) \
  94   _ (AES_128_CBC, SHA256, "aes-128-cbc-hmac-sha-256", 16, 16) \
  95   _ (AES_192_CBC, SHA256, "aes-192-cbc-hmac-sha-256", 24, 16) \
  96   _ (AES_256_CBC, SHA256, "aes-256-cbc-hmac-sha-256", 32, 16) \
  97   _ (AES_128_CBC, SHA384, "aes-128-cbc-hmac-sha-384", 16, 24) \
  98   _ (AES_192_CBC, SHA384, "aes-192-cbc-hmac-sha-384", 24, 24) \
  99   _ (AES_256_CBC, SHA384, "aes-256-cbc-hmac-sha-384", 32, 24) \
 100   _ (AES_128_CBC, SHA512, "aes-128-cbc-hmac-sha-512", 16, 32) \
 101   _ (AES_192_CBC, SHA512, "aes-192-cbc-hmac-sha-512", 24, 32) \
 102   _ (AES_256_CBC, SHA512, "aes-256-cbc-hmac-sha-512", 32, 32)
 103
 104 #define foreach_crypto_async_op_type \
 105   _(ENCRYPT, "async-encrypt") \
 106   _(DECRYPT, "async-decrypt")
 107
 108 typedef enum
 109 {
 110   VNET_CRYPTO_KEY_OP_ADD,
 111   VNET_CRYPTO_KEY_OP_DEL,
 112   VNET_CRYPTO_KEY_OP_MODIFY,
 113 } vnet_crypto_key_op_t;
 114
 115 typedef enum
 116 {
 117 #define _(n, s) VNET_CRYPTO_OP_STATUS_##n,
 118   foreach_crypto_op_status
 119 #undef _
 120     VNET_CRYPTO_OP_N_STATUS,
 121 } vnet_crypto_op_status_t;
 122
 123 /* *INDENT-OFF* */
 124 typedef enum
 125 {
 126   VNET_CRYPTO_ALG_NONE = 0,
 127 #define _(n, s, l) VNET_CRYPTO_ALG_##n,
 128   foreach_crypto_cipher_alg
 129   foreach_crypto_aead_alg
 130 #undef _
 131 #define _(n, s) VNET_CRYPTO_ALG_HMAC_##n,
 132   foreach_crypto_hmac_alg
 133 #undef _
 134   VNET_CRYPTO_N_ALGS,
 135 } vnet_crypto_alg_t;
 136
 137 typedef enum
 138 {
 139 #define _(n, s) VNET_CRYPTO_ASYNC_OP_TYPE_##n,
 140   foreach_crypto_async_op_type
 141 #undef _
 142     VNET_CRYPTO_ASYNC_OP_N_TYPES,
 143 } vnet_crypto_async_op_type_t;
 144
 145 typedef enum
 146 {
 147   VNET_CRYPTO_ASYNC_ALG_NONE = 0,
 148 #define _(n, s, k, t, a) \
 149   VNET_CRYPTO_ALG_##n##_TAG##t##_AAD##a,
 150   foreach_crypto_aead_async_alg
 151 #undef _
 152 #define _(c, h, s, k ,d) \
 153   VNET_CRYPTO_ALG_##c##_##h##_TAG##d,
 154   foreach_crypto_link_async_alg
 155 #undef _
 156   VNET_CRYPTO_N_ASYNC_ALGS,
 157 } vnet_crypto_async_alg_t;
 158
 159 typedef enum
 160 {
 161   VNET_CRYPTO_ASYNC_OP_NONE = 0,
 162 #define _(n, s, k, t, a) \
 163   VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \
 164   VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC,
 165   foreach_crypto_aead_async_alg
 166 #undef _
 167 #define _(c, h, s, k ,d) \
 168   VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC, \
 169   VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC,
 170   foreach_crypto_link_async_alg
 171 #undef _
 172   VNET_CRYPTO_ASYNC_OP_N_IDS,
 173 } vnet_crypto_async_op_id_t;
 174
 175 typedef struct
 176 {
 177   union
 178   {
 179     struct
 180     {
 181       u8 *data;
 182       vnet_crypto_alg_t alg:8;
 183     };
 184     struct
 185     {
 186       u32 index_crypto;
 187       u32 index_integ;
 188       vnet_crypto_async_alg_t async_alg:8;
 189     };
 190   };
 191 #define VNET_CRYPTO_KEY_TYPE_DATA 0
 192 #define VNET_CRYPTO_KEY_TYPE_LINK 1
 193   u8 type;
 194 } vnet_crypto_key_t;
 195
 196 typedef enum
 197 {
 198   VNET_CRYPTO_OP_NONE = 0,
 199 #define _(n, s, l) VNET_CRYPTO_OP_##n##_ENC, VNET_CRYPTO_OP_##n##_DEC,
 200   foreach_crypto_cipher_alg
 201   foreach_crypto_aead_alg
 202 #undef _
 203 #define _(n, s) VNET_CRYPTO_OP_##n##_HMAC,
 204  foreach_crypto_hmac_alg
 205 #undef _
 206     VNET_CRYPTO_N_OP_IDS,
 207 } vnet_crypto_op_id_t;
 208 /* *INDENT-ON* */
 209
 210 typedef enum
 211 {
 212   CRYPTO_OP_SIMPLE,
 213   CRYPTO_OP_CHAINED,
 214   CRYPTO_OP_BOTH,
 215 } crypto_op_class_type_t;
 216
 217 typedef struct
 218 {
 219   char *name;
 220   vnet_crypto_op_id_t op_by_type[VNET_CRYPTO_OP_N_TYPES];
 221 } vnet_crypto_alg_data_t;
 222
 223 typedef struct
 224 {
 225   u8 *src;
 226   u8 *dst;
 227   u32 len;
 228 } vnet_crypto_op_chunk_t;
 229
 230 typedef struct
 231 {
 232   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 233   uword user_data;
 234   vnet_crypto_op_id_t op:16;
 235   vnet_crypto_op_status_t status:8;
 236   u8 flags;
 237 #define VNET_CRYPTO_OP_FLAG_INIT_IV (1 << 0)
 238 #define VNET_CRYPTO_OP_FLAG_HMAC_CHECK (1 << 1)
 239 #define VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS (1 << 2)
 240
 241   union
 242   {
 243     u8 digest_len;
 244     u8 tag_len;
 245   };
 246   u16 aad_len;
 247
 248   union
 249   {
 250     struct
 251     {
 252       u8 *src;
 253       u8 *dst;
 254     };
 255
 256     /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */
 257     u16 n_chunks;
 258   };
 259
 260   union
 261   {
 262     u32 len;
 263     /* valid if VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS is set */
 264     u32 chunk_index;
 265   };
 266
 267   u32 key_index;
 268   u8 *iv;
 269   u8 *aad;
 270
 271   union
 272   {
 273     u8 *tag;
 274     u8 *digest;
 275   };
 276 } vnet_crypto_op_t;
 277
 278 STATIC_ASSERT_SIZEOF (vnet_crypto_op_t, CLIB_CACHE_LINE_BYTES);
 279
 280 typedef struct
 281 {
 282   vnet_crypto_op_type_t type;
 283   vnet_crypto_alg_t alg;
 284   u32 active_engine_index_simple;
 285   u32 active_engine_index_chained;
 286 } vnet_crypto_op_data_t;
 287
 288 typedef struct
 289 {
 290   vnet_crypto_async_op_type_t type;
 291   vnet_crypto_async_alg_t alg;
 292   u32 active_engine_index_async;
 293 } vnet_crypto_async_op_data_t;
 294
 295 typedef struct
 296 {
 297   char *name;
 298   vnet_crypto_async_op_id_t op_by_type[VNET_CRYPTO_ASYNC_OP_N_TYPES];
 299 } vnet_crypto_async_alg_data_t;
 300
 301 typedef struct
 302 {
 303   vnet_crypto_op_status_t status:8;
 304   u32 key_index;
 305   i16 crypto_start_offset;      /* first buffer offset */
 306   i16 integ_start_offset;
 307   u32 crypto_total_length;
 308   /* adj total_length for integ, e.g.4 bytes for IPSec ESN */
 309   u16 integ_length_adj;
 310   u8 *iv;
 311   union
 312   {
 313     u8 *digest;
 314     u8 *tag;
 315   };
 316   u8 *aad;
 317   u8 flags; /**< share same VNET_CRYPTO_OP_FLAG_* values */
 318 } vnet_crypto_async_frame_elt_t;
 319
 320 typedef struct
 321 {
 322   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 323 #define VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED 0
 324 #define VNET_CRYPTO_FRAME_STATE_PENDING 1       /* frame waiting to be processed */
 325 #define VNET_CRYPTO_FRAME_STATE_WORK_IN_PROGRESS 2
 326 #define VNET_CRYPTO_FRAME_STATE_SUCCESS 3
 327 #define VNET_CRYPTO_FRAME_STATE_ELT_ERROR 4
 328   u8 state;
 329   vnet_crypto_async_op_id_t op:8;
 330   u16 n_elts;
 331   vnet_crypto_async_frame_elt_t elts[VNET_CRYPTO_FRAME_SIZE];
 332   u32 buffer_indices[VNET_CRYPTO_FRAME_SIZE];
 333   u16 next_node_index[VNET_CRYPTO_FRAME_SIZE];
 334   u32 enqueue_thread_index;
 335 } vnet_crypto_async_frame_t;
 336
 337 typedef struct
 338 {
 339   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 340   vnet_crypto_async_frame_t *frames[VNET_CRYPTO_ASYNC_OP_N_IDS];
 341   vnet_crypto_async_frame_t *frame_pool;
 342   u32 *buffer_indice;
 343   u16 *nexts;
 344 } vnet_crypto_thread_t;
 345
 346 typedef u32 vnet_crypto_key_index_t;
 347
 348 typedef u32 (vnet_crypto_chained_ops_handler_t) (vlib_main_t * vm,
 349                                                  vnet_crypto_op_t * ops[],
 350                                                  vnet_crypto_op_chunk_t *
 351                                                  chunks, u32 n_ops);
 352
 353 typedef u32 (vnet_crypto_ops_handler_t) (vlib_main_t * vm,
 354                                          vnet_crypto_op_t * ops[], u32 n_ops);
 355
 356 typedef void (vnet_crypto_key_handler_t) (vlib_main_t * vm,
 357                                           vnet_crypto_key_op_t kop,
 358                                           vnet_crypto_key_index_t idx);
 359
 360 /** async crypto function handlers **/
 361 typedef int
 362   (vnet_crypto_frame_enqueue_t) (vlib_main_t * vm,
 363                                  vnet_crypto_async_frame_t * frame);
 364 typedef vnet_crypto_async_frame_t *
 365   (vnet_crypto_frame_dequeue_t) (vlib_main_t * vm, u32 * nb_elts_processed,
 366                                  u32 * enqueue_thread_idx);
 367
 368 u32
 369 vnet_crypto_register_engine (vlib_main_t * vm, char *name, int prio,
 370                              char *desc);
 371
 372 void vnet_crypto_register_ops_handler (vlib_main_t * vm, u32 engine_index,
 373                                        vnet_crypto_op_id_t opt,
 374                                        vnet_crypto_ops_handler_t * oph);
 375
 376 void vnet_crypto_register_chained_ops_handler (vlib_main_t * vm,
 377                                                u32 engine_index,
 378                                                vnet_crypto_op_id_t opt,
 379                                                vnet_crypto_chained_ops_handler_t
 380                                                * oph);
 381
 382 void vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index,
 383                                         vnet_crypto_op_id_t opt,
 384                                         vnet_crypto_ops_handler_t * fn,
 385                                         vnet_crypto_chained_ops_handler_t *
 386                                         cfn);
 387
 388 void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
 389                                        vnet_crypto_key_handler_t * keyh);
 390
 391 /** async crypto register functions */
 392 u32 vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name);
 393 void vnet_crypto_register_async_handler (vlib_main_t * vm,
 394                                          u32 engine_index,
 395                                          vnet_crypto_async_op_id_t opt,
 396                                          vnet_crypto_frame_enqueue_t * enq_fn,
 397                                          vnet_crypto_frame_dequeue_t *
 398                                          deq_fn);
 399
 400 typedef struct
 401 {
 402   char *name;
 403   char *desc;
 404   int priority;
 405   vnet_crypto_key_handler_t *key_op_handler;
 406   vnet_crypto_ops_handler_t *ops_handlers[VNET_CRYPTO_N_OP_IDS];
 407     vnet_crypto_chained_ops_handler_t
 408     * chained_ops_handlers[VNET_CRYPTO_N_OP_IDS];
 409   vnet_crypto_frame_enqueue_t *enqueue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
 410   vnet_crypto_frame_dequeue_t *dequeue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
 411 } vnet_crypto_engine_t;
 412
 413 typedef struct
 414 {
 415   u32 node_idx;
 416   u32 next_idx;
 417 } vnet_crypto_async_next_node_t;
 418
 419 typedef struct
 420 {
 421   vnet_crypto_alg_data_t *algs;
 422   vnet_crypto_thread_t *threads;
 423   vnet_crypto_ops_handler_t **ops_handlers;
 424   vnet_crypto_chained_ops_handler_t **chained_ops_handlers;
 425   vnet_crypto_frame_enqueue_t **enqueue_handlers;
 426   vnet_crypto_frame_dequeue_t **dequeue_handlers;
 427   clib_bitmap_t *async_active_ids;
 428   vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS];
 429   vnet_crypto_async_op_data_t async_opt_data[VNET_CRYPTO_ASYNC_OP_N_IDS];
 430   vnet_crypto_engine_t *engines;
 431   vnet_crypto_key_t *keys;
 432   uword *engine_index_by_name;
 433   uword *alg_index_by_name;
 434   uword *async_alg_index_by_name;
 435   vnet_crypto_async_alg_data_t *async_algs;
 436   u32 async_refcnt;
 437   vnet_crypto_async_next_node_t *next_nodes;
 438   u32 crypto_node_index;
 439 #define VNET_CRYPTO_ASYNC_DISPATCH_POLLING 0
 440 #define VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT 1
 441   u8 dispatch_mode;
 442 } vnet_crypto_main_t;
 443
 444 extern vnet_crypto_main_t crypto_main;
 445
 446 u32 vnet_crypto_process_chained_ops (vlib_main_t * vm, vnet_crypto_op_t ops[],
 447                                      vnet_crypto_op_chunk_t * chunks,
 448                                      u32 n_ops);
 449 u32 vnet_crypto_process_ops (vlib_main_t * vm, vnet_crypto_op_t ops[],
 450                              u32 n_ops);
 451
 452
 453 int vnet_crypto_set_handler2 (char *ops_handler_name, char *engine,
 454                               crypto_op_class_type_t oct);
 455 int vnet_crypto_is_set_handler (vnet_crypto_alg_t alg);
 456
 457 u32 vnet_crypto_key_add (vlib_main_t * vm, vnet_crypto_alg_t alg,
 458                          u8 * data, u16 length);
 459 void vnet_crypto_key_del (vlib_main_t * vm, vnet_crypto_key_index_t index);
 460
 461 /**
 462  * Use 2 created keys to generate new key for linked algs (cipher + integ)
 463  * The returned key index is to be used for linked alg only.
 464  **/
 465 u32 vnet_crypto_key_add_linked (vlib_main_t * vm,
 466                                 vnet_crypto_key_index_t index_crypto,
 467                                 vnet_crypto_key_index_t index_integ);
 468
 469 clib_error_t *crypto_dispatch_enable_disable (int is_enable);
 470
 471 int vnet_crypto_set_async_handler2 (char *alg_name, char *engine);
 472
 473 int vnet_crypto_is_set_async_handler (vnet_crypto_async_op_id_t opt);
 474
 475 void vnet_crypto_request_async_mode (int is_enable);
 476
 477 void vnet_crypto_set_async_dispatch_mode (u8 mode);
 478
 479 vnet_crypto_async_alg_t vnet_crypto_link_algs (vnet_crypto_alg_t crypto_alg,
 480                                                vnet_crypto_alg_t integ_alg);
 481
 482 clib_error_t *crypto_dispatch_enable_disable (int is_enable);
 483
 484 format_function_t format_vnet_crypto_alg;
 485 format_function_t format_vnet_crypto_engine;
 486 format_function_t format_vnet_crypto_op;
 487 format_function_t format_vnet_crypto_op_type;
 488 format_function_t format_vnet_crypto_op_status;
 489 unformat_function_t unformat_vnet_crypto_alg;
 490
 491 format_function_t format_vnet_crypto_async_op;
 492 format_function_t format_vnet_crypto_async_alg;
 493 format_function_t format_vnet_crypto_async_op_type;
 494
 495 static_always_inline void
 496 vnet_crypto_op_init (vnet_crypto_op_t * op, vnet_crypto_op_id_t type)
 497 {
 498   if (CLIB_DEBUG > 0)
 499     clib_memset (op, 0xfe, sizeof (*op));
 500   op->op = type;
 501   op->flags = 0;
 502   op->key_index = ~0;
 503   op->n_chunks = 0;
 504 }
 505
 506 static_always_inline vnet_crypto_op_type_t
 507 vnet_crypto_get_op_type (vnet_crypto_op_id_t id)
 508 {
 509   vnet_crypto_main_t *cm = &crypto_main;
 510   ASSERT (id < VNET_CRYPTO_N_OP_IDS);
 511   vnet_crypto_op_data_t *od = cm->opt_data + id;
 512   return od->type;
 513 }
 514
 515 static_always_inline vnet_crypto_key_t *
 516 vnet_crypto_get_key (vnet_crypto_key_index_t index)
 517 {
 518   vnet_crypto_main_t *cm = &crypto_main;
 519   return vec_elt_at_index (cm->keys, index);
 520 }
 521
 522 static_always_inline int
 523 vnet_crypto_set_handler (char *alg_name, char *engine)
 524 {
 525   return vnet_crypto_set_handler2 (alg_name, engine, CRYPTO_OP_BOTH);
 526 }
 527
 528 /** async crypto inline functions **/
 529
 530 static_always_inline vnet_crypto_async_frame_t *
 531 vnet_crypto_async_get_frame (vlib_main_t * vm, vnet_crypto_async_op_id_t opt)
 532 {
 533   vnet_crypto_main_t *cm = &crypto_main;
 534   vnet_crypto_thread_t *ct = cm->threads + vm->thread_index;
 535   vnet_crypto_async_frame_t *f = ct->frames[opt];
 536
 537   if (!f)
 538     {
 539       pool_get_aligned (ct->frame_pool, f, CLIB_CACHE_LINE_BYTES);
 540       if (CLIB_DEBUG > 0)
 541         clib_memset (f, 0xfe, sizeof (*f));
 542       f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
 543       f->op = opt;
 544       f->n_elts = 0;
 545       ct->frames[opt] = f;
 546     }
 547   return f;
 548 }
 549
 550 static_always_inline void
 551 vnet_crypto_async_free_frame (vlib_main_t * vm,
 552                               vnet_crypto_async_frame_t * frame)
 553 {
 554   vnet_crypto_main_t *cm = &crypto_main;
 555   vnet_crypto_thread_t *ct = cm->threads + vm->thread_index;
 556   pool_put (ct->frame_pool, frame);
 557 }
 558
 559 static_always_inline int
 560 vnet_crypto_async_submit_open_frame (vlib_main_t * vm,
 561                                      vnet_crypto_async_frame_t * frame)
 562 {
 563   vnet_crypto_main_t *cm = &crypto_main;
 564   vlib_thread_main_t *tm = vlib_get_thread_main ();
 565   vnet_crypto_thread_t *ct = cm->threads + vm->thread_index;
 566   vnet_crypto_async_op_id_t opt = frame->op;
 567   u32 i = vlib_num_workers () > 0;
 568
 569   frame->state = VNET_CRYPTO_FRAME_STATE_PENDING;
 570   frame->enqueue_thread_index = vm->thread_index;
 571
 572   int ret = (cm->enqueue_handlers[frame->op]) (vm, frame);
 573
 574   clib_bitmap_set_no_check (cm->async_active_ids, opt, 1);
 575   if (PREDICT_TRUE (ret == 0))
 576     {
 577       vnet_crypto_async_frame_t *nf = 0;
 578       pool_get_aligned (ct->frame_pool, nf, CLIB_CACHE_LINE_BYTES);
 579       if (CLIB_DEBUG > 0)
 580         clib_memset (nf, 0xfe, sizeof (*nf));
 581       nf->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
 582       nf->op = opt;
 583       nf->n_elts = 0;
 584       ct->frames[opt] = nf;
 585     }
 586   else
 587     {
 588       frame->state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
 589     }
 590
 591   if (cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT)
 592     {
 593       for (; i < tm->n_vlib_mains; i++)
 594         {
 595           vlib_node_set_interrupt_pending (vlib_mains[i],
 596                                            cm->crypto_node_index);
 597         }
 598     }
 599   return ret;
 600 }
 601
 602 static_always_inline int
 603 vnet_crypto_async_add_to_frame (vlib_main_t * vm,
 604                                 vnet_crypto_async_frame_t ** frame,
 605                                 u32 key_index,
 606                                 u32 crypto_len, i16 integ_len_adj,
 607                                 i16 crypto_start_offset,
 608                                 u16 integ_start_offset,
 609                                 u32 buffer_index,
 610                                 u16 next_node,
 611                                 u8 * iv, u8 * tag, u8 * aad, u8 flags)
 612 {
 613   vnet_crypto_async_frame_t *f = *frame;
 614   vnet_crypto_async_frame_elt_t *fe;
 615   u16 index;
 616
 617   if (PREDICT_FALSE (f->n_elts == VNET_CRYPTO_FRAME_SIZE))
 618     {
 619       vnet_crypto_async_op_id_t opt = f->op;
 620       int ret;
 621       ret = vnet_crypto_async_submit_open_frame (vm, f);
 622       if (PREDICT_FALSE (ret < 0))
 623         return -1;
 624       f = vnet_crypto_async_get_frame (vm, opt);
 625       *frame = f;
 626     }
 627
 628   index = f->n_elts;
 629   fe = &f->elts[index];
 630   f->n_elts++;
 631   fe->key_index = key_index;
 632   fe->crypto_total_length = crypto_len;
 633   fe->crypto_start_offset = crypto_start_offset;
 634   fe->integ_start_offset = integ_start_offset;
 635   fe->integ_length_adj = integ_len_adj;
 636   fe->iv = iv;
 637   fe->tag = tag;
 638   fe->aad = aad;
 639   fe->flags = flags;
 640   f->buffer_indices[index] = buffer_index;
 641   f->next_node_index[index] = next_node;
 642
 643   return 0;
 644 }
 645
 646 static_always_inline void
 647 vnet_crypto_async_reset_frame (vnet_crypto_async_frame_t * f)
 648 {
 649   vnet_crypto_async_op_id_t opt;
 650   ASSERT (f != 0);
 651   ASSERT (f->state == VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
 652   opt = f->op;
 653   if (CLIB_DEBUG > 0)
 654     clib_memset (f, 0xfe, sizeof (*f));
 655   f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
 656   f->op = opt;
 657   f->n_elts = 0;
 658 }
 659
 660 #endif /* included_vnet_crypto_crypto_h */
 661
 662 /*
 663  * fd.io coding-style-patch-verification: ON
 664  *
 665  * Local Variables:
 666  * eval: (c-set-style "gnu")
 667  * End:
 668  */