2 * Copyright (c) 2017 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #ifndef __DPDK_IPSEC_H__
16 #define __DPDK_IPSEC_H__
18 #include <vnet/vnet.h>
19 #include <vppinfra/cache.h>
20 #include <vnet/ipsec/ipsec.h>
21 #include <vnet/ipsec/ipsec_sa.h>
24 #include <rte_config.h>
25 #include <rte_crypto.h>
26 #include <rte_cryptodev.h>
29 #define always_inline static inline
31 #define always_inline static inline __attribute__ ((__always_inline__))
34 #define DPDK_CRYPTO_N_QUEUE_DESC 2048
35 #define DPDK_CRYPTO_NB_SESS_OBJS 20000
37 #define foreach_dpdk_crypto_input_next \
38 _(DROP, "error-drop") \
39 _(IP4_LOOKUP, "ip4-lookup") \
40 _(IP6_LOOKUP, "ip6-lookup") \
41 _(INTERFACE_OUTPUT, "interface-output") \
42 _(MIDCHAIN, "adj-midchain-tx") \
43 _(DECRYPT4_POST, "dpdk-esp4-decrypt-post") \
44 _(DECRYPT6_POST, "dpdk-esp6-decrypt-post")
48 #define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
49 foreach_dpdk_crypto_input_next
51 DPDK_CRYPTO_INPUT_N_NEXT,
52 } dpdk_crypto_input_next_t;
54 #define MAX_QP_PER_LCORE 16
68 CLIB_ALIGN_MARK (mark0, 16);
71 u8 icv[32]; /* XXX last 16B in next cache line */
77 struct rte_crypto_op **ops;
78 u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG];
79 u16 auth_resource_idx[IPSEC_INTEG_N_ALG];
80 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
81 } crypto_worker_main_t;
85 CLIB_ALIGN_MARK (pad, 8); /* align up to 8 bytes for 32bit builds */
87 enum rte_crypto_sym_xform_type type;
101 u8 cipher_support[IPSEC_CRYPTO_N_ALG];
102 u8 auth_support[IPSEC_INTEG_N_ALG];
128 struct rte_crypto_op *ops[VLIB_FRAME_SIZE];
129 u32 bi[VLIB_FRAME_SIZE];
130 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
136 struct rte_cryptodev_sym_session *session;
137 } crypto_session_disposal_t;
141 struct rte_cryptodev_sym_session *session;
143 CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
144 } crypto_session_by_drv_t;
148 struct rte_mempool *crypto_op;
149 struct rte_mempool *session_h;
150 struct rte_mempool **session_drv;
151 crypto_session_disposal_t *session_disposal;
152 uword *session_by_sa_index;
153 u64 crypto_op_get_failed;
154 u64 session_h_failed;
155 u64 *session_drv_failed;
156 crypto_session_by_drv_t *session_by_drv_id_and_sa_index;
157 clib_spinlock_t lockp;
158 /* Required for vec_validate_aligned */
159 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
164 crypto_worker_main_t *workers_main;
166 crypto_resource_t *resource;
167 crypto_alg_t *cipher_algs;
168 crypto_alg_t *auth_algs;
171 u64 session_timeout; /* nsec */
173 } dpdk_crypto_main_t;
175 extern dpdk_crypto_main_t dpdk_crypto_main;
177 static const u8 pad_data[] =
178 { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 };
180 void crypto_auto_placement (void);
182 clib_error_t *create_sym_session (struct rte_cryptodev_sym_session **session,
183 u32 sa_idx, crypto_resource_t * res,
184 crypto_worker_main_t * cwm, u8 is_outbound);
186 static_always_inline u32
191 sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
193 return ((op_size + align - 1) & ~(align - 1)) + sizeof (dpdk_op_priv_t);
196 static_always_inline u32
197 crypto_op_get_priv_offset (void)
199 const u32 align = 16;
202 offset = sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
203 offset = (offset + align - 1) & ~(align - 1);
208 static_always_inline dpdk_op_priv_t *
209 crypto_op_get_priv (struct rte_crypto_op * op)
211 return (dpdk_op_priv_t *) (((u8 *) op) + crypto_op_get_priv_offset ());
215 static_always_inline void
216 add_session_by_drv_and_sa_idx (struct rte_cryptodev_sym_session *session,
217 crypto_data_t * data, u32 drv_id, u32 sa_idx)
219 crypto_session_by_drv_t *sbd;
220 vec_validate_aligned (data->session_by_drv_id_and_sa_index, sa_idx,
221 CLIB_CACHE_LINE_BYTES);
222 sbd = vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
223 sbd->dev_mask |= 1L << drv_id;
224 sbd->session = session;
227 static_always_inline struct rte_cryptodev_sym_session *
228 get_session_by_drv_and_sa_idx (crypto_data_t * data, u32 drv_id, u32 sa_idx)
230 crypto_session_by_drv_t *sess_by_sa;
231 if (_vec_len (data->session_by_drv_id_and_sa_index) <= sa_idx)
234 vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
235 return (sess_by_sa->dev_mask & (1L << drv_id)) ? sess_by_sa->session : NULL;
238 static_always_inline clib_error_t *
239 crypto_get_session (struct rte_cryptodev_sym_session ** session,
241 crypto_resource_t * res,
242 crypto_worker_main_t * cwm, u8 is_outbound)
244 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
246 struct rte_cryptodev_sym_session *sess;
248 data = vec_elt_at_index (dcm->data, res->numa);
249 sess = get_session_by_drv_and_sa_idx (data, res->drv_id, sa_idx);
251 if (PREDICT_FALSE (!sess))
252 return create_sym_session (session, sa_idx, res, cwm, is_outbound);
259 static_always_inline u16
260 get_resource (crypto_worker_main_t * cwm, ipsec_sa_t * sa)
262 u16 cipher_res = cwm->cipher_resource_idx[sa->crypto_alg];
263 u16 auth_res = cwm->auth_resource_idx[sa->integ_alg];
266 /* Not allowed to setup SA with no-aead-cipher/NULL or NULL/NULL */
268 is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) ||
269 (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) ||
270 (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256));
272 if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE)
275 if (cipher_res == auth_res)
284 static_always_inline i32
285 crypto_alloc_ops (u8 numa, struct rte_crypto_op ** ops, u32 n)
287 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
288 crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
291 ret = rte_mempool_get_bulk (data->crypto_op, (void **) ops, n);
294 data->crypto_op_get_failed += ! !ret;
300 static_always_inline void
301 crypto_free_ops (u8 numa, struct rte_crypto_op **ops, u32 n)
303 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
304 crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
309 rte_mempool_put_bulk (data->crypto_op, (void **) ops, n);
312 static_always_inline void
313 crypto_enqueue_ops (vlib_main_t * vm, crypto_worker_main_t * cwm,
314 u32 node_index, u32 error, u8 numa, u8 encrypt)
316 dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
317 crypto_resource_t *res;
321 vec_foreach (res_idx, cwm->resource_idx)
324 res = vec_elt_at_index (dcm->resource, res_idx[0]);
329 n_ops = (DPDK_CRYPTO_N_QUEUE_DESC / 2) - res->inflights[encrypt];
330 n_ops = res->n_ops < n_ops ? res->n_ops : n_ops;
331 enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id,
333 ASSERT (n_ops == enq);
334 res->inflights[encrypt] += enq;
336 if (PREDICT_FALSE (enq < res->n_ops))
338 crypto_free_ops (numa, &res->ops[enq], res->n_ops - enq);
339 vlib_buffer_free (vm, &res->bi[enq], res->n_ops - enq);
341 vlib_node_increment_counter (vm, node_index, error,
349 static_always_inline void
350 crypto_set_icb (dpdk_gcm_cnt_blk * icb, u32 salt, u32 seq, u32 seq_hi)
357 static_always_inline void
358 crypto_op_setup (u8 is_aead, struct rte_mbuf *mb0,
359 struct rte_crypto_op *op, void *session,
360 u32 cipher_off, u32 cipher_len,
361 u32 auth_off, u32 auth_len,
362 u8 * aad, u8 * digest, u64 digest_paddr)
364 struct rte_crypto_sym_op *sym_op;
366 sym_op = (struct rte_crypto_sym_op *) (op + 1);
369 sym_op->session = session;
373 sym_op->aead.data.offset = cipher_off;
374 sym_op->aead.data.length = cipher_len;
376 sym_op->aead.aad.data = aad;
377 sym_op->aead.aad.phys_addr =
378 op->phys_addr + (uintptr_t) aad - (uintptr_t) op;
380 sym_op->aead.digest.data = digest;
381 sym_op->aead.digest.phys_addr = digest_paddr;
385 sym_op->cipher.data.offset = cipher_off;
386 sym_op->cipher.data.length = cipher_len;
388 sym_op->auth.data.offset = auth_off;
389 sym_op->auth.data.length = auth_len;
391 sym_op->auth.digest.data = digest;
392 sym_op->auth.digest.phys_addr = digest_paddr;
396 #endif /* __DPDK_IPSEC_H__ */
399 * fd.io coding-style-patch-verification: ON
402 * eval: (c-set-style "gnu")