X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fipsec%2Fesp_decrypt.c;h=f0fa506debb25c92ecfceba03b50044d5ec72f1f;hb=b4d3053;hp=bab76e3b665f3af6f73c80a2a7df9f278fdcf35c;hpb=586afd762bfa149f5ca167bd5fd5a0cd59ce94fe;p=vpp.git diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index bab76e3b665..f0fa506debb 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -1,10 +1,10 @@ /* * esp_decrypt.c : IPSec ESP Decrypt node using DPDK Cryptodev * - * Copyright (c) 2016 Intel and/or its affiliates. + * Copyright (c) 2017 Intel and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: + * You may obtain a opy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -20,14 +20,14 @@ #include #include +#include #include -#include #include #include #define foreach_esp_decrypt_next \ _(DROP, "error-drop") \ -_(IP4_INPUT, "ip4-input") \ +_(IP4_INPUT, "ip4-input-no-checksum") \ _(IP6_INPUT, "ip6-input") #define _(v, s) ESP_DECRYPT_NEXT_##v, @@ -43,9 +43,10 @@ typedef enum { _(REPLAY, "SA replayed packet") \ _(NOT_IP, "Not IP packet (dropped)") \ _(ENQ_FAIL, "Enqueue failed (buffer full)") \ - _(NO_CRYPTODEV, "Cryptodev not configured") \ + _(DISCARD, "Not enough crypto operations, discarding frame") \ _(BAD_LEN, "Invalid ciphertext length") \ - _(UNSUPPORTED, "Cipher/Auth not supported") + _(SESSION, "Failed to get crypto session") \ + _(NOSUP, "Cipher/Auth not supported") typedef enum { @@ -61,11 +62,13 @@ static char * esp_decrypt_error_strings[] = { #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_node; +vlib_node_registration_t dpdk_esp4_decrypt_node; +vlib_node_registration_t dpdk_esp6_decrypt_node; typedef struct { ipsec_crypto_alg_t crypto_alg; ipsec_integ_alg_t integ_alg; + u8 packet_data[64]; } esp_decrypt_trace_t; /* packet trace format function */ @@ -74,51 +77,54 @@ static u8 * format_esp_decrypt_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); + u32 indent = format_get_indent (s); - s = format (s, "esp: crypto %U integrity %U", + s = format (s, "cipher %U auth %U\n", format_ipsec_crypto_alg, t->crypto_alg, format_ipsec_integ_alg, t->integ_alg); + s = format (s, "%U%U", + format_white_space, indent, + format_esp_header, t->packet_data); return s; } -static uword -dpdk_esp_decrypt_node_fn (vlib_main_t * vm, +always_inline uword +dpdk_esp_decrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) + vlib_frame_t * from_frame, + int is_ip6) { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; - u32 thread_index = vlib_get_thread_index(); - dpdk_crypto_main_t * dcm = &dpdk_crypto_main; - dpdk_esp_main_t * em = &dpdk_esp_main; - u32 i; + u32 thread_idx = vlib_get_thread_index(); + dpdk_crypto_main_t *dcm = &dpdk_crypto_main; + crypto_resource_t *res = 0; + ipsec_sa_t *sa0 = 0; + crypto_alg_t *cipher_alg = 0, *auth_alg = 0; + struct rte_cryptodev_sym_session *session = 0; + u32 ret, last_sa_index = ~0; + u8 numa = rte_socket_id (); + u8 is_aead = 0; + crypto_worker_main_t *cwm = + vec_elt_at_index (dcm->workers_main, thread_idx); + struct rte_crypto_op **ops = cwm->ops; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; - if (PREDICT_FALSE(!dcm->workers_main)) + ret = crypto_alloc_ops (numa, ops, n_left_from); + if (ret) { - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_NO_CRYPTODEV, n_left_from); - vlib_buffer_free(vm, from, n_left_from); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DISCARD, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_DISCARD, 1); + /* Discard whole frame */ return n_left_from; } - crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, thread_index); - u32 n_qps = vec_len(cwm->qp_data); - struct rte_crypto_op ** cops_to_enq[n_qps]; - u32 n_cop_qp[n_qps], * bi_to_enq[n_qps]; - - for (i = 0; i < n_qps; i++) - { - bi_to_enq[i] = cwm->qp_data[i].bi; - cops_to_enq[i] = cwm->qp_data[i].cops; - } - - memset(n_cop_qp, 0, n_qps * sizeof(u32)); - - crypto_alloc_cops(); - next_index = ESP_DECRYPT_NEXT_DROP; while (n_left_from > 0) @@ -129,73 +135,109 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, while (n_left_from > 0 && n_left_to_next > 0) { - u32 bi0, sa_index0 = ~0, seq, icv_size, iv_size; - vlib_buffer_t * b0; - esp_header_t * esp0; - ipsec_sa_t * sa0; - struct rte_mbuf * mb0 = 0; - const int BLOCK_SIZE = 16; - crypto_sa_session_t * sa_sess; - void * sess; - u16 qp_index; - struct rte_crypto_op * cop = 0; + clib_error_t *error; + u32 bi0, sa_index0, seq, iv_size; + u8 trunc_size; + vlib_buffer_t *b0; + esp_header_t *esp0; + struct rte_mbuf *mb0; + struct rte_crypto_op *op; + u16 res_idx; bi0 = from[0]; from += 1; n_left_from -= 1; b0 = vlib_get_buffer (vm, bi0); + mb0 = rte_mbuf_from_vlib_buffer(b0); esp0 = vlib_buffer_get_current (b0); - sa_index0 = vnet_buffer(b0)->ipsec.sad_index; - sa0 = pool_elt_at_index (im->sad, sa_index0); + /* ih0/ih6_0 */ + CLIB_PREFETCH (esp0, sizeof (esp0[0]) + 16, LOAD); + /* mb0 */ + CLIB_PREFETCH (mb0, CLIB_CACHE_LINE_BYTES, STORE); - seq = clib_host_to_net_u32(esp0->seq); + op = ops[0]; + ops += 1; + ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED); - /* anti-replay check */ - if (sa0->use_anti_replay) + dpdk_op_priv_t *priv = crypto_op_get_priv (op); + + u16 op_len = + sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]); + CLIB_PREFETCH (op, op_len, STORE); + + sa_index0 = vnet_buffer(b0)->ipsec.sad_index; + + if (sa_index0 != last_sa_index) { - int rv = 0; + sa0 = pool_elt_at_index (im->sad, sa_index0); - if (PREDICT_TRUE(sa0->use_esn)) - rv = esp_replay_check_esn(sa0, seq); - else - rv = esp_replay_check(sa0, seq); + cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); + auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); + + is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD); + if (is_aead) + auth_alg = cipher_alg; + + res_idx = get_resource (cwm, sa0); - if (PREDICT_FALSE(rv)) + if (PREDICT_FALSE (res_idx == (u16) ~0)) { - clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_REPLAY, 1); + clib_warning ("unsupported SA by thread index %u", thread_idx); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_NOSUP, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_NOSUP, 1); to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; goto trace; } - } + res = vec_elt_at_index (dcm->resource, res_idx); - sa0->total_data_size += b0->current_length; + error = crypto_get_session (&session, sa_index0, res, cwm, 0); + if (PREDICT_FALSE (error || !session)) + { + clib_warning ("failed to get crypto session"); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_SESSION, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_SESSION, 1); + to_next[0] = bi0; + to_next += 1; + n_left_to_next -= 1; + goto trace; + } - if (PREDICT_FALSE(sa0->integ_alg == IPSEC_INTEG_ALG_NONE) || - PREDICT_FALSE(sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE)) - { - clib_warning ("SPI %u : only cipher + auth supported", sa0->spi); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_UNSUPPORTED, 1); - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; + last_sa_index = sa_index0; } - sa_sess = pool_elt_at_index(cwm->sa_sess_d[0], sa_index0); - - if (PREDICT_FALSE(!sa_sess->sess)) + /* anti-replay check */ + if (sa0->use_anti_replay) { - int ret = create_sym_sess(sa0, sa_sess, 0); + int rv = 0; - if (PREDICT_FALSE (ret)) + seq = clib_net_to_host_u32 (esp0->seq); + + if (PREDICT_TRUE(sa0->use_esn)) + rv = esp_replay_check_esn (sa0, seq); + else + rv = esp_replay_check (sa0, seq); + + if (PREDICT_FALSE (rv)) { + clib_warning ("failed anti-replay check"); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_REPLAY, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_REPLAY, 1); to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; @@ -203,172 +245,150 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, } } - sess = sa_sess->sess; - qp_index = sa_sess->qp_index; - - ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0); - cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops); - ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED); - - cops_to_enq[qp_index][0] = cop; - cops_to_enq[qp_index] += 1; - n_cop_qp[qp_index] += 1; - bi_to_enq[qp_index][0] = bi0; - bi_to_enq[qp_index] += 1; + if(is_ip6) + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT6_POST; + else + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT4_POST; - rte_crypto_op_attach_sym_session(cop, sess); + /* FIXME multi-seg */ + sa0->total_data_size += b0->current_length; - icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size; - iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len; + res->ops[res->n_ops] = op; + res->bi[res->n_ops] = bi0; + res->n_ops += 1; /* Convert vlib buffer to mbuf */ - mb0 = rte_mbuf_from_vlib_buffer(b0); mb0->data_len = b0->current_length; mb0->pkt_len = b0->current_length; mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data; + trunc_size = auth_alg->trunc_size; + iv_size = cipher_alg->iv_len; + /* Outer IP header has already been stripped */ - u16 payload_len = rte_pktmbuf_pkt_len(mb0) - sizeof (esp_header_t) - - iv_size - icv_size; + u16 payload_len = + b0->current_length - sizeof (esp_header_t) - iv_size - trunc_size; + + ASSERT (payload_len >= 4); - if ((payload_len & (BLOCK_SIZE - 1)) || (payload_len <= 0)) + if (payload_len & (cipher_alg->boundary - 1)) { clib_warning ("payload %u not multiple of %d\n", - payload_len, BLOCK_SIZE); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + payload_len, cipher_alg->boundary); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_BAD_LEN, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_BAD_LEN, 1); - vec_add (vec_elt (cwm->qp_data, qp_index).free_cops, &cop, 1); - bi_to_enq[qp_index] -= 1; - cops_to_enq[qp_index] -= 1; - n_cop_qp[qp_index] -= 1; + res->n_ops -= 1; to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; goto trace; } - struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1); + u32 cipher_off, cipher_len; + u32 auth_len = 0; + u8 *aad = NULL; - sym_cop->m_src = mb0; - sym_cop->cipher.data.offset = sizeof (esp_header_t) + iv_size; - sym_cop->cipher.data.length = payload_len; + u8 *iv = (u8 *) (esp0 + 1); - u8 *iv = rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t)); - dpdk_cop_priv_t * priv = (dpdk_cop_priv_t *)(sym_cop + 1); + dpdk_gcm_cnt_blk *icb = &priv->cb; - if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) - { - dpdk_gcm_cnt_blk *icb = &priv->cb; - icb->salt = sa0->salt; - clib_memcpy(icb->iv, iv, 8); - icb->cnt = clib_host_to_net_u32(1); - sym_cop->cipher.iv.data = (u8 *)icb; - sym_cop->cipher.iv.phys_addr = cop->phys_addr + - (uintptr_t)icb - (uintptr_t)cop; - sym_cop->cipher.iv.length = 16; - - u8 *aad = priv->aad; - clib_memcpy(aad, iv - sizeof(esp_header_t), 8); - sym_cop->auth.aad.data = aad; - sym_cop->auth.aad.phys_addr = cop->phys_addr + - (uintptr_t)aad - (uintptr_t)cop; - if (sa0->use_esn) - { - *((u32*)&aad[8]) = sa0->seq_hi; - sym_cop->auth.aad.length = 12; - } - else - { - sym_cop->auth.aad.length = 8; - } + cipher_off = sizeof (esp_header_t) + iv_size; + cipher_len = payload_len; + + u8 *digest = vlib_buffer_get_tail (b0) - trunc_size; + u64 digest_paddr = + mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr); + + if (!is_aead && cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC) + clib_memcpy_fast(icb, iv, 16); + else /* CTR/GCM */ + { + u32 *_iv = (u32 *) iv; - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.length = icv_size; + crypto_set_icb (icb, sa0->salt, _iv[0], _iv[1]); + } + + if (is_aead) + { + aad = priv->aad; + u32 * _aad = (u32 *) aad; + clib_memcpy_fast (aad, esp0, 8); + /* _aad[3] should always be 0 */ + if (PREDICT_FALSE (sa0->use_esn)) + _aad[2] = clib_host_to_net_u32 (sa0->seq_hi); + else + _aad[2] = 0; } else { - sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(mb0, void*, - sizeof (esp_header_t)); - sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - sizeof (esp_header_t)); - sym_cop->cipher.iv.length = iv_size; + auth_len = sizeof(esp_header_t) + iv_size + payload_len; if (sa0->use_esn) { - dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1); - u8* payload_end = rte_pktmbuf_mtod_offset( - mb0, u8*, sizeof(esp_header_t) + iv_size + payload_len); - - clib_memcpy (priv->icv, payload_end, icv_size); - *((u32*) payload_end) = sa0->seq_hi; - sym_cop->auth.data.offset = 0; - sym_cop->auth.data.length = sizeof(esp_header_t) + iv_size - + payload_len + sizeof(sa0->seq_hi); - sym_cop->auth.digest.data = priv->icv; - sym_cop->auth.digest.phys_addr = cop->phys_addr - + (uintptr_t) priv->icv - (uintptr_t) cop; - sym_cop->auth.digest.length = icv_size; - } - else - { - sym_cop->auth.data.offset = 0; - sym_cop->auth.data.length = sizeof(esp_header_t) + - iv_size + payload_len; - - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.length = icv_size; + clib_memcpy_fast (priv->icv, digest, trunc_size); + u32 *_digest = (u32 *) digest; + _digest[0] = clib_host_to_net_u32 (sa0->seq_hi); + auth_len += sizeof(sa0->seq_hi); + + digest = priv->icv; + digest_paddr = + op->phys_addr + (uintptr_t) priv->icv - (uintptr_t) op; } } + crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len, + 0, auth_len, aad, digest, digest_paddr); trace: if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b0), + sizeof (esp_header_t)); } } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_RX_PKTS, - from_frame->n_vectors); - crypto_qp_data_t *qpd; - /* *INDENT-OFF* */ - vec_foreach_index (i, cwm->qp_data) - { - u32 enq; - qpd = vec_elt_at_index(cwm->qp_data, i); - enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id, - qpd->cops, n_cop_qp[i]); - qpd->inflights += enq; + if(is_ip6){ + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); - if (PREDICT_FALSE(enq < n_cop_qp[i])) - { - crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq); - vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq); + crypto_enqueue_ops (vm, cwm, 0, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_ENQ_FAIL, numa); + } + else + { + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_ENQ_FAIL, - n_cop_qp[i] - enq); - } + crypto_enqueue_ops (vm, cwm, 0, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_ENQ_FAIL, numa); } - /* *INDENT-ON* */ + + crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops); return from_frame->n_vectors; } -VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { - .function = dpdk_esp_decrypt_node_fn, - .name = "dpdk-esp-decrypt", +VLIB_NODE_FN(dpdk_esp4_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline(vm, node, from_frame, 0 /*is_ip6*/); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_node) = { + .name = "dpdk-esp4-decrypt", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -383,8 +403,33 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { #undef _ }, }; +/* *INDENT-ON* */ + +VLIB_NODE_FN(dpdk_esp6_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline(vm, node, from_frame, 1 /*is_ip6*/); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_node) = { + .name = "dpdk-esp6-decrypt", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_trace, + .type = VLIB_NODE_TYPE_INTERNAL, -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn) + .n_errors = ARRAY_LEN(esp_decrypt_error_strings), + .error_strings = esp_decrypt_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */ /* * Decrypt Post Node @@ -406,23 +451,40 @@ static char * esp_decrypt_post_error_strings[] = { #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_post_node; +vlib_node_registration_t dpdk_esp4_decrypt_post_node; +vlib_node_registration_t dpdk_esp6_decrypt_post_node; static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) { + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); + u32 indent = format_get_indent (s); + + s = format (s, "cipher %U auth %U\n", + format_ipsec_crypto_alg, t->crypto_alg, + format_ipsec_integ_alg, t->integ_alg); + + ip4_header_t *ih4 = (ip4_header_t *) t->packet_data; + if ((ih4->ip_version_and_header_length & 0xF0) == 0x60) + s = format (s, "%U%U", format_white_space, indent, format_ip6_header, ih4); + else + s = format (s, "%U%U", format_white_space, indent, format_ip4_header, ih4); + return s; } -static uword -dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, +always_inline uword +dpdk_esp_decrypt_post_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) + vlib_frame_t * from_frame, + int is_ip6) { u32 n_left_from, *from, *to_next = 0, next_index; ipsec_sa_t * sa0; u32 sa_index0 = ~0; ipsec_main_t *im = &ipsec_main; - dpdk_esp_main_t *em = &dpdk_esp_main; + dpdk_crypto_main_t *dcm = &dpdk_crypto_main; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -438,12 +500,14 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, while (n_left_from > 0 && n_left_to_next > 0) { esp_footer_t * f0; - u32 bi0, next0, icv_size, iv_size; + u32 bi0, iv_size, next0; vlib_buffer_t * b0 = 0; ip4_header_t *ih4 = 0, *oh4 = 0; ip6_header_t *ih6 = 0, *oh6 = 0; - u8 tunnel_mode = 1; - u8 transport_ip6 = 0; + crypto_alg_t *cipher_alg, *auth_alg; + esp_header_t *esp0; + u8 trunc_size, is_aead; + u16 udp_encap_adv = 0; next0 = ESP_DECRYPT_NEXT_DROP; @@ -453,6 +517,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); + esp0 = vlib_buffer_get_current (b0); sa_index0 = vnet_buffer(b0)->ipsec.sad_index; sa0 = pool_elt_at_index (im->sad, sa_index0); @@ -460,12 +525,18 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, to_next[0] = bi0; to_next += 1; - icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size; - iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len; + cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); + auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); + is_aead = cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD; + if (is_aead) + auth_alg = cipher_alg; + + trunc_size = auth_alg->trunc_size; + + iv_size = cipher_alg->iv_len; if (sa0->use_anti_replay) { - esp_header_t * esp0 = vlib_buffer_get_current (b0); u32 seq; seq = clib_host_to_net_u32(esp0->seq); if (PREDICT_TRUE(sa0->use_esn)) @@ -474,91 +545,109 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, esp_replay_advance(sa0, seq); } - ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t)); - vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size); + /* if UDP encapsulation is used adjust the address of the IP header */ + if (sa0->udp_encap && (b0->flags & VNET_BUFFER_F_IS_IP4)) + { + udp_encap_adv = sizeof (udp_header_t); + } - b0->current_length -= (icv_size + 2); - b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; - f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (b0) + - b0->current_length); - b0->current_length -= f0->pad_length; + if (b0->flags & VNET_BUFFER_F_IS_IP4) + ih4 = (ip4_header_t *) + ((u8 *) esp0 - udp_encap_adv - sizeof (ip4_header_t)); + else + ih4 = + (ip4_header_t *) ((u8 *) esp0 - sizeof (ip6_header_t)); - /* transport mode */ - if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6)) - { - tunnel_mode = 0; + vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size); - if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) != 0x40)) - { - if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) == 0x60)) - transport_ip6 = 1; - else - { - clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_NOT_IP, 1); - goto trace; - } - } + b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + f0 = (esp_footer_t *) (vlib_buffer_get_tail (b0) - trunc_size - 2); + b0->current_length -= (f0->pad_length + trunc_size + 2); +#if 0 + /* check padding */ + const u8 *padding = vlib_buffer_get_tail (b0); + if (PREDICT_FALSE (memcmp (padding, pad_data, f0->pad_length))) + { + clib_warning("bad padding"); + vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + goto trace; } - - if (PREDICT_TRUE (tunnel_mode)) +#endif + if (sa0->is_tunnel) { - if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP)) + if (f0->next_header == IP_PROTOCOL_IP_IN_IP) next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - else if (f0->next_header == IP_PROTOCOL_IPV6) + else if (sa0->is_tunnel_ip6 && f0->next_header == IP_PROTOCOL_IPV6) next0 = ESP_DECRYPT_NEXT_IP6_INPUT; else { clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_DECRYPTION_FAILED, 1); goto trace; } } - /* transport mode */ - else + else /* transport mode */ { - if (PREDICT_FALSE(transport_ip6)) + if ((ih4->ip_version_and_header_length & 0xF0) == 0x40) { - ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t)); + u16 ih4_len = ip4_header_bytes (ih4); + vlib_buffer_advance (b0, - ih4_len - udp_encap_adv); + next0 = ESP_DECRYPT_NEXT_IP4_INPUT; + if (!sa0->udp_encap) + { + oh4 = vlib_buffer_get_current (b0); + memmove(oh4, ih4, ih4_len); + oh4->protocol = f0->next_header; + oh4->length = clib_host_to_net_u16 (b0->current_length); + oh4->checksum = ip4_header_checksum(oh4); + } + } + else if ((ih4->ip_version_and_header_length & 0xF0) == 0x60) + { + ih6 = (ip6_header_t *) ih4; vlib_buffer_advance (b0, -sizeof(ip6_header_t)); oh6 = vlib_buffer_get_current (b0); memmove(oh6, ih6, sizeof(ip6_header_t)); next0 = ESP_DECRYPT_NEXT_IP6_INPUT; oh6->protocol = f0->next_header; - oh6->payload_length = - clib_host_to_net_u16 ( - vlib_buffer_length_in_chain(vm, b0) - - sizeof (ip6_header_t)); + u16 len = b0->current_length - sizeof (ip6_header_t); + oh6->payload_length = clib_host_to_net_u16 (len); } else { - vlib_buffer_advance (b0, -sizeof(ip4_header_t)); - oh4 = vlib_buffer_get_current (b0); - memmove(oh4, ih4, sizeof(ip4_header_t)); - - next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - oh4->ip_version_and_header_length = 0x45; - oh4->fragment_id = 0; - oh4->flags_and_fragment_offset = 0; - oh4->protocol = f0->next_header; - oh4->length = clib_host_to_net_u16 ( - vlib_buffer_length_in_chain (vm, b0)); - oh4->checksum = ip4_header_checksum (oh4); + clib_warning("next header: 0x%x", f0->next_header); + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + goto trace; } } vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0; -trace: + trace: if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; + ih4 = vlib_buffer_get_current (b0); + clib_memcpy_fast (tr->packet_data, ih4, sizeof (ip6_header_t)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -566,16 +655,27 @@ trace: } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index, + + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_post_node.index, + ESP_DECRYPT_POST_ERROR_PKTS, + from_frame->n_vectors); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_post_node.index, ESP_DECRYPT_POST_ERROR_PKTS, from_frame->n_vectors); return from_frame->n_vectors; } -VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { - .function = dpdk_esp_decrypt_post_node_fn, - .name = "dpdk-esp-decrypt-post", +VLIB_NODE_FN(dpdk_esp4_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{return dpdk_esp_decrypt_post_inline(vm, node, from_frame, 0/*is_ip6*/);} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_post_node) = { + .name = "dpdk-esp4-decrypt-post", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_post_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -590,5 +690,28 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { #undef _ }, }; +/* *INDENT-ON* */ + +VLIB_NODE_FN(dpdk_esp6_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{return dpdk_esp_decrypt_post_inline(vm, node, from_frame, 0/*is_ip6*/);} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_post_node) = { + .name = "dpdk-esp6-decrypt-post", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_post_trace, + .type = VLIB_NODE_TYPE_INTERNAL, -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn) + .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings), + .error_strings = esp_decrypt_post_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */