X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fipsec%2Fesp_decrypt.c;h=1ed35188e185d212c2a6ac2215b5369084dbd495;hb=040950a59d53e8802ad31430d67df105939cce4c;hp=36be24ba430d937fe61170bb8f9fa0ea7be61317;hpb=2096063b0856808326cbd2c1c8a3a786c98ea896;p=vpp.git diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 36be24ba430..1ed35188e18 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -62,7 +62,8 @@ static char * esp_decrypt_error_strings[] = { #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_node; +extern vlib_node_registration_t dpdk_esp4_decrypt_node; +extern vlib_node_registration_t dpdk_esp6_decrypt_node; typedef struct { ipsec_crypto_alg_t crypto_alg; @@ -76,7 +77,7 @@ static u8 * format_esp_decrypt_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "cipher %U auth %U\n", format_ipsec_crypto_alg, t->crypto_alg, @@ -87,10 +88,11 @@ static u8 * format_esp_decrypt_trace (u8 * s, va_list * args) return s; } -static uword -dpdk_esp_decrypt_node_fn (vlib_main_t * vm, +always_inline uword +dpdk_esp_decrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) + vlib_frame_t * from_frame, + int is_ip6) { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; @@ -113,7 +115,11 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, ret = crypto_alloc_ops (numa, ops, n_left_from); if (ret) { - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DISCARD, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_DISCARD, 1); /* Discard whole frame */ return n_left_from; @@ -170,13 +176,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); -#if DPDK_NO_AEAD - is_aead = (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256); -#else is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD); -#endif if (is_aead) auth_alg = cipher_alg; @@ -185,7 +185,11 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (res_idx == (u16) ~0)) { clib_warning ("unsupported SA by thread index %u", thread_idx); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_NOSUP, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_NOSUP, 1); to_next[0] = bi0; to_next += 1; @@ -198,7 +202,11 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (error || !session)) { clib_warning ("failed to get crypto session"); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_SESSION, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_SESSION, 1); to_next[0] = bi0; to_next += 1; @@ -224,7 +232,11 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (rv)) { clib_warning ("failed anti-replay check"); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_REPLAY, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_REPLAY, 1); to_next[0] = bi0; to_next += 1; @@ -233,7 +245,10 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, } } - priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT_POST; + if(is_ip6) + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT6_POST; + else + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT4_POST; /* FIXME multi-seg */ sa0->total_data_size += b0->current_length; @@ -260,7 +275,11 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, { clib_warning ("payload %u not multiple of %d\n", payload_len, cipher_alg->boundary); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_BAD_LEN, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_BAD_LEN, 1); res->n_ops -= 1; to_next[0] = bi0; @@ -270,9 +289,8 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, } u32 cipher_off, cipher_len; - u32 auth_len = 0, aad_size = 0; - u8 *aad = NULL, *digest = NULL; - u64 digest_paddr = 0; + u32 auth_len = 0; + u8 *aad = NULL; u8 *iv = (u8 *) (esp0 + 1); @@ -281,33 +299,30 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, cipher_off = sizeof (esp_header_t) + iv_size; cipher_len = payload_len; - digest = vlib_buffer_get_tail (b0) - trunc_size; + u8 *digest = vlib_buffer_get_tail (b0) - trunc_size; + u64 digest_paddr = + mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr); if (!is_aead && cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC) - clib_memcpy(icb, iv, 16); + clib_memcpy_fast(icb, iv, 16); else /* CTR/GCM */ { u32 *_iv = (u32 *) iv; crypto_set_icb (icb, sa0->salt, _iv[0], _iv[1]); -#if DPDK_NO_AEAD - iv_size = 16; -#else - iv_size = 12; -#endif } if (is_aead) { aad = priv->aad; - clib_memcpy(aad, esp0, 8); + u32 * _aad = (u32 *) aad; + clib_memcpy_fast (aad, esp0, 8); + + /* _aad[3] should always be 0 */ if (PREDICT_FALSE (sa0->use_esn)) - { - *((u32*)&aad[8]) = sa0->seq_hi; - aad_size = 12; - } + _aad[2] = clib_host_to_net_u32 (sa0->seq_hi); else - aad_size = 8; + _aad[2] = 0; } else { @@ -315,8 +330,9 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, if (sa0->use_esn) { - clib_memcpy (priv->icv, digest, trunc_size); - *((u32*) digest) = sa0->seq_hi; + clib_memcpy_fast (priv->icv, digest, trunc_size); + u32 *_digest = (u32 *) digest; + _digest[0] = clib_host_to_net_u32 (sa0->seq_hi); auth_len += sizeof(sa0->seq_hi); digest = priv->icv; @@ -325,39 +341,54 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, } } - crypto_op_setup (is_aead, mb0, op, session, - cipher_off, cipher_len, (u8 *) icb, iv_size, - 0, auth_len, aad, aad_size, - digest, digest_paddr, trunc_size); + crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len, + 0, auth_len, aad, digest, digest_paddr); trace: if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b0), + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b0), sizeof (esp_header_t)); } } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_RX_PKTS, - from_frame->n_vectors); + if(is_ip6){ + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); - crypto_enqueue_ops (vm, cwm, 0, dpdk_esp_decrypt_node.index, + crypto_enqueue_ops (vm, cwm, 0, dpdk_esp6_decrypt_node.index, ESP_DECRYPT_ERROR_ENQ_FAIL, numa); + } + else + { + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); + + crypto_enqueue_ops (vm, cwm, 0, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_ENQ_FAIL, numa); + } crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops); return from_frame->n_vectors; } +VLIB_NODE_FN(dpdk_esp4_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline(vm, node, from_frame, 0 /*is_ip6*/); +} + /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { - .function = dpdk_esp_decrypt_node_fn, - .name = "dpdk-esp-decrypt", +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_node) = { + .name = "dpdk-esp4-decrypt", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -374,7 +405,31 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn) +VLIB_NODE_FN(dpdk_esp6_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline(vm, node, from_frame, 1 /*is_ip6*/); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_node) = { + .name = "dpdk-esp6-decrypt", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(esp_decrypt_error_strings), + .error_strings = esp_decrypt_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */ /* * Decrypt Post Node @@ -396,14 +451,15 @@ static char * esp_decrypt_post_error_strings[] = { #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_post_node; +extern vlib_node_registration_t dpdk_esp4_decrypt_post_node; +extern vlib_node_registration_t dpdk_esp6_decrypt_post_node; static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "cipher %U auth %U\n", format_ipsec_crypto_alg, t->crypto_alg, @@ -418,10 +474,11 @@ static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) return s; } -static uword -dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, +always_inline uword +dpdk_esp_decrypt_post_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) + vlib_frame_t * from_frame, + int is_ip6) { u32 n_left_from, *from, *to_next = 0, next_index; ipsec_sa_t * sa0; @@ -450,6 +507,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, crypto_alg_t *cipher_alg, *auth_alg; esp_header_t *esp0; u8 trunc_size, is_aead; + u16 udp_encap_adv = 0; next0 = ESP_DECRYPT_NEXT_DROP; @@ -469,13 +527,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); -#if DPDK_NO_AEAD - is_aead = (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256); -#else is_aead = cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD; -#endif if (is_aead) auth_alg = cipher_alg; @@ -493,8 +545,19 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, esp_replay_advance(sa0, seq); } - /* FIXME ip header */ - ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t)); + /* if UDP encapsulation is used adjust the address of the IP header */ + if (sa0->udp_encap && (b0->flags & VNET_BUFFER_F_IS_IP4)) + { + udp_encap_adv = sizeof (udp_header_t); + } + + if (b0->flags & VNET_BUFFER_F_IS_IP4) + ih4 = (ip4_header_t *) + ((u8 *) esp0 - udp_encap_adv - sizeof (ip4_header_t)); + else + ih4 = + (ip4_header_t *) ((u8 *) esp0 - sizeof (ip6_header_t)); + vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size); b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -521,7 +584,12 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, else { clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_DECRYPTION_FAILED, 1); goto trace; @@ -531,28 +599,21 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, { if ((ih4->ip_version_and_header_length & 0xF0) == 0x40) { - u16 ih4_len = ip4_header_bytes (ih4); - vlib_buffer_advance (b0, - ih4_len); - oh4 = vlib_buffer_get_current (b0); - memmove(oh4, ih4, ih4_len); - - next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - u16 old_ttl_prot = - ((u16) oh4->ttl) << 8 | (u16) oh4->protocol; - u16 new_ttl_prot = - ((u16) oh4->ttl) << 8 | (u16) f0->next_header; - oh4->protocol = f0->next_header; - u16 new_len = clib_host_to_net_u16 (b0->current_length); - oh4->length = new_len; - /* rfc1264 incremental checksum update */ - oh4->checksum = ~(~oh4->checksum + ~oh4->length + new_len + - ~old_ttl_prot + new_ttl_prot); - + u16 ih4_len = ip4_header_bytes (ih4); + vlib_buffer_advance (b0, - ih4_len - udp_encap_adv); + next0 = ESP_DECRYPT_NEXT_IP4_INPUT; + if (!sa0->udp_encap) + { + oh4 = vlib_buffer_get_current (b0); + memmove(oh4, ih4, ih4_len); + oh4->protocol = f0->next_header; + oh4->length = clib_host_to_net_u16 (b0->current_length); + oh4->checksum = ip4_header_checksum(oh4); + } } else if ((ih4->ip_version_and_header_length & 0xF0) == 0x60) { - /* FIXME find ip header */ - ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t)); + ih6 = (ip6_header_t *) ih4; vlib_buffer_advance (b0, -sizeof(ip6_header_t)); oh6 = vlib_buffer_get_current (b0); memmove(oh6, ih6, sizeof(ip6_header_t)); @@ -565,7 +626,12 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, else { clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, ESP_DECRYPT_ERROR_DECRYPTION_FAILED, 1); goto trace; @@ -581,7 +647,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; ih4 = vlib_buffer_get_current (b0); - clib_memcpy (tr->packet_data, ih4, sizeof (ip6_header_t)); + clib_memcpy_fast (tr->packet_data, ih4, sizeof (ip6_header_t)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -590,17 +656,52 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index, + if(is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_post_node.index, + ESP_DECRYPT_POST_ERROR_PKTS, + from_frame->n_vectors); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_post_node.index, ESP_DECRYPT_POST_ERROR_PKTS, from_frame->n_vectors); return from_frame->n_vectors; } +VLIB_NODE_FN(dpdk_esp4_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{return dpdk_esp_decrypt_post_inline(vm, node, from_frame, 0/*is_ip6*/);} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_post_node) = { + .name = "dpdk-esp4-decrypt-post", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_post_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings), + .error_strings = esp_decrypt_post_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +VLIB_NODE_FN(dpdk_esp6_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_post_inline(vm, node, from_frame, 0/*is_ip6*/); +} + /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { - .function = dpdk_esp_decrypt_post_node_fn, - .name = "dpdk-esp-decrypt-post", +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_post_node) = { + .name = "dpdk-esp6-decrypt-post", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_post_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -617,4 +718,3 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn)