X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fdpdk%2Fipsec%2Fesp_decrypt.c;h=ae35ab5aaae3fabbc45e3efb6db5fd18d828c66e;hb=82fc98fa4578dbbfb156effb11dea6a4e2d0b898;hp=90be466efd7dc075eaa83c1ac34b0ecbeb57472b;hpb=db93cd971320301eb21403caabada7a3ec6a4cce;p=vpp.git diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 90be466efd7..ae35ab5aaae 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -31,10 +32,11 @@ _(IP4_INPUT, "ip4-input-no-checksum") \ _(IP6_INPUT, "ip6-input") #define _(v, s) ESP_DECRYPT_NEXT_##v, -typedef enum { +typedef enum +{ foreach_esp_decrypt_next #undef _ - ESP_DECRYPT_N_NEXT, + ESP_DECRYPT_N_NEXT, } esp_decrypt_next_t; #define foreach_esp_decrypt_error \ @@ -49,52 +51,55 @@ typedef enum { _(NOSUP, "Cipher/Auth not supported") -typedef enum { +typedef enum +{ #define _(sym,str) ESP_DECRYPT_ERROR_##sym, foreach_esp_decrypt_error #undef _ - ESP_DECRYPT_N_ERROR, + ESP_DECRYPT_N_ERROR, } esp_decrypt_error_t; -static char * esp_decrypt_error_strings[] = { +static char *esp_decrypt_error_strings[] = { #define _(sym,string) string, foreach_esp_decrypt_error #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_node; +extern vlib_node_registration_t dpdk_esp4_decrypt_node; +extern vlib_node_registration_t dpdk_esp6_decrypt_node; -typedef struct { +typedef struct +{ ipsec_crypto_alg_t crypto_alg; ipsec_integ_alg_t integ_alg; u8 packet_data[64]; } esp_decrypt_trace_t; /* packet trace format function */ -static u8 * format_esp_decrypt_trace (u8 * s, va_list * args) +static u8 * +format_esp_decrypt_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); - uword indent = format_get_indent (s); + esp_decrypt_trace_t *t = va_arg (*args, esp_decrypt_trace_t *); + u32 indent = format_get_indent (s); s = format (s, "cipher %U auth %U\n", format_ipsec_crypto_alg, t->crypto_alg, format_ipsec_integ_alg, t->integ_alg); s = format (s, "%U%U", - format_white_space, indent, - format_esp_header, t->packet_data); + format_white_space, indent, format_esp_header, t->packet_data); return s; } -static uword -dpdk_esp_decrypt_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +always_inline uword +dpdk_esp_decrypt_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, int is_ip6) { - u32 n_left_from, *from, *to_next, next_index; + u32 n_left_from, *from, *to_next, next_index, thread_index; ipsec_main_t *im = &ipsec_main; - u32 thread_idx = vlib_get_thread_index(); + u32 thread_idx = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; crypto_resource_t *res = 0; ipsec_sa_t *sa0 = 0; @@ -109,12 +114,17 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; + thread_index = vm->thread_index; ret = crypto_alloc_ops (numa, ops, n_left_from); if (ret) { - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_DISCARD, 1); + if (is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DISCARD, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_DISCARD, 1); /* Discard whole frame */ return n_left_from; } @@ -130,7 +140,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, while (n_left_from > 0 && n_left_to_next > 0) { clib_error_t *error; - u32 bi0, sa_index0, seq, iv_size; + u32 bi0, sa_index0, iv_size; u8 trunc_size; vlib_buffer_t *b0; esp_header_t *esp0; @@ -143,7 +153,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, n_left_from -= 1; b0 = vlib_get_buffer (vm, bi0); - mb0 = rte_mbuf_from_vlib_buffer(b0); + mb0 = rte_mbuf_from_vlib_buffer (b0); esp0 = vlib_buffer_get_current (b0); /* ih0/ih6_0 */ @@ -156,39 +166,44 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, ASSERT (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED); dpdk_op_priv_t *priv = crypto_op_get_priv (op); + /* store bi in op private */ + priv->bi = bi0; + priv->encrypt = 0; u16 op_len = sizeof (op[0]) + sizeof (op[0].sym[0]) + sizeof (priv[0]); CLIB_PREFETCH (op, op_len, STORE); - sa_index0 = vnet_buffer(b0)->ipsec.sad_index; + sa_index0 = vnet_buffer (b0)->ipsec.sad_index; + vlib_prefetch_combined_counter (&ipsec_sa_counters, + thread_index, sa_index0); if (sa_index0 != last_sa_index) { - last_sa_index = sa_index0; - sa0 = pool_elt_at_index (im->sad, sa_index0); - cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); + cipher_alg = + vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); -#if DPDK_NO_AEAD - is_aead = (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256); -#else is_aead = (cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD); -#endif if (is_aead) auth_alg = cipher_alg; res_idx = get_resource (cwm, sa0); - if (PREDICT_FALSE (res_idx == (u16) ~0)) + if (PREDICT_FALSE (res_idx == (u16) ~ 0)) { - clib_warning ("unsupported SA by thread index %u", thread_idx); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_NOSUP, 1); + clib_warning ("unsupported SA by thread index %u", + thread_idx); + if (is_ip6) + vlib_node_increment_counter (vm, + dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_NOSUP, 1); + else + vlib_node_increment_counter (vm, + dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_NOSUP, 1); to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; @@ -200,43 +215,56 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (error || !session)) { clib_warning ("failed to get crypto session"); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_SESSION, 1); + if (is_ip6) + vlib_node_increment_counter (vm, + dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_SESSION, + 1); + else + vlib_node_increment_counter (vm, + dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_SESSION, + 1); to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; goto trace; } + + last_sa_index = sa_index0; } /* anti-replay check */ - if (sa0->use_anti_replay) + if (ipsec_sa_anti_replay_check + (sa0, clib_host_to_net_u32 (esp0->seq))) { - int rv = 0; - - seq = clib_net_to_host_u32 (esp0->seq); - - if (PREDICT_TRUE(sa0->use_esn)) - rv = esp_replay_check_esn (sa0, seq); + clib_warning ("failed anti-replay check"); + if (is_ip6) + vlib_node_increment_counter (vm, + dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_REPLAY, 1); else - rv = esp_replay_check (sa0, seq); - - if (PREDICT_FALSE (rv)) - { - clib_warning ("failed anti-replay check"); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_REPLAY, 1); - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; - } + vlib_node_increment_counter (vm, + dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_REPLAY, 1); + to_next[0] = bi0; + to_next += 1; + n_left_to_next -= 1; + goto trace; } - priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT_POST; + if (is_ip6) + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT6_POST; + else + { + priv->next = DPDK_CRYPTO_INPUT_NEXT_DECRYPT4_POST; + b0->flags |= VNET_BUFFER_F_IS_IP4; + } /* FIXME multi-seg */ - sa0->total_data_size += b0->current_length; + vlib_increment_combined_counter + (&ipsec_sa_counters, thread_index, sa_index0, + 1, b0->current_length); res->ops[res->n_ops] = op; res->bi[res->n_ops] = bi0; @@ -260,8 +288,12 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, { clib_warning ("payload %u not multiple of %d\n", payload_len, cipher_alg->boundary); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_BAD_LEN, 1); + if (is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_BAD_LEN, 1); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_BAD_LEN, 1); res->n_ops -= 1; to_next[0] = bi0; to_next += 1; @@ -270,94 +302,111 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, } u32 cipher_off, cipher_len; - u32 auth_len = 0, aad_size = 0; - u8 *aad = NULL, *digest = NULL; - u64 digest_paddr = 0; + u32 auth_len = 0; + u8 *aad = NULL; - u8 *iv = (u8 *) (esp0 + 1); + u8 *iv = (u8 *) (esp0 + 1); - dpdk_gcm_cnt_blk *icb = &priv->cb; + dpdk_gcm_cnt_blk *icb = &priv->cb; cipher_off = sizeof (esp_header_t) + iv_size; cipher_len = payload_len; - digest = vlib_buffer_get_tail (b0) - trunc_size; + u8 *digest = vlib_buffer_get_tail (b0) - trunc_size; + u64 digest_paddr = + mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr); - if (cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC) - clib_memcpy(icb, iv, 16); - else /* CTR/GCM */ + if (!is_aead && cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC) + clib_memcpy_fast (icb, iv, 16); + else /* CTR/GCM */ { u32 *_iv = (u32 *) iv; crypto_set_icb (icb, sa0->salt, _iv[0], _iv[1]); -#if DPDK_NO_AEAD - iv_size = 16; -#else - iv_size = 12; -#endif } - if (is_aead) - { - aad = priv->aad; - clib_memcpy(aad, esp0, 8); - if (PREDICT_FALSE (sa0->use_esn)) + if (is_aead) + { + aad = priv->aad; + u32 *_aad = (u32 *) aad; + clib_memcpy_fast (aad, esp0, 8); + + /* _aad[3] should always be 0 */ + if (PREDICT_FALSE (ipsec_sa_is_set_USE_ESN (sa0))) { - *((u32*)&aad[8]) = sa0->seq_hi; - aad_size = 12; + _aad[2] = _aad[1]; + _aad[1] = clib_host_to_net_u32 (sa0->seq_hi); } else - aad_size = 8; - } - else - { - auth_len = sizeof(esp_header_t) + iv_size + payload_len; - - if (sa0->use_esn) - { - clib_memcpy (priv->icv, digest, trunc_size); - *((u32*) digest) = sa0->seq_hi; - auth_len += sizeof(sa0->seq_hi); - - digest = priv->icv; + _aad[2] = 0; + } + else + { + auth_len = sizeof (esp_header_t) + iv_size + payload_len; + + if (ipsec_sa_is_set_USE_ESN (sa0)) + { + clib_memcpy_fast (priv->icv, digest, trunc_size); + u32 *_digest = (u32 *) digest; + _digest[0] = clib_host_to_net_u32 (sa0->seq_hi); + auth_len += sizeof (sa0->seq_hi); + + digest = priv->icv; digest_paddr = op->phys_addr + (uintptr_t) priv->icv - (uintptr_t) op; - } - } - - crypto_op_setup (is_aead, mb0, op, session, - cipher_off, cipher_len, (u8 *) icb, iv_size, - 0, auth_len, aad, aad_size, - digest, digest_paddr, trunc_size); -trace: - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + } + } + + crypto_op_setup (is_aead, mb0, op, session, cipher_off, cipher_len, + 0, auth_len, aad, digest, digest_paddr); + trace: + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); + esp_decrypt_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b0), - sizeof (esp_header_t)); + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b0), + sizeof (esp_header_t)); } } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_RX_PKTS, - from_frame->n_vectors); + if (is_ip6) + { + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); + + crypto_enqueue_ops (vm, cwm, dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_ENQ_FAIL, numa, 0 /* encrypt */ ); + } + else + { + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_RX_PKTS, + from_frame->n_vectors); - crypto_enqueue_ops (vm, cwm, 0, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_ENQ_FAIL, numa); + crypto_enqueue_ops (vm, cwm, dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_ENQ_FAIL, numa, 0 /* encrypt */ ); + } crypto_free_ops (numa, ops, cwm->ops + from_frame->n_vectors - ops); return from_frame->n_vectors; } +VLIB_NODE_FN (dpdk_esp4_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline (vm, node, from_frame, 0 /*is_ip6 */ ); +} + /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { - .function = dpdk_esp_decrypt_node_fn, - .name = "dpdk-esp-decrypt", +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_node) = { + .name = "dpdk-esp4-decrypt", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -374,7 +423,31 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn) +VLIB_NODE_FN (dpdk_esp6_decrypt_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_inline (vm, node, from_frame, 1 /*is_ip6 */ ); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_node) = { + .name = "dpdk-esp6-decrypt", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(esp_decrypt_error_strings), + .error_strings = esp_decrypt_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */ /* * Decrypt Post Node @@ -383,27 +456,30 @@ VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn) #define foreach_esp_decrypt_post_error \ _(PKTS, "ESP post pkts") -typedef enum { +typedef enum +{ #define _(sym,str) ESP_DECRYPT_POST_ERROR_##sym, foreach_esp_decrypt_post_error #undef _ - ESP_DECRYPT_POST_N_ERROR, + ESP_DECRYPT_POST_N_ERROR, } esp_decrypt_post_error_t; -static char * esp_decrypt_post_error_strings[] = { +static char *esp_decrypt_post_error_strings[] = { #define _(sym,string) string, foreach_esp_decrypt_post_error #undef _ }; -vlib_node_registration_t dpdk_esp_decrypt_post_node; +extern vlib_node_registration_t dpdk_esp4_decrypt_post_node; +extern vlib_node_registration_t dpdk_esp6_decrypt_post_node; -static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) +static u8 * +format_esp_decrypt_post_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); - uword indent = format_get_indent (s); + esp_decrypt_trace_t *t = va_arg (*args, esp_decrypt_trace_t *); + u32 indent = format_get_indent (s); s = format (s, "cipher %U auth %U\n", format_ipsec_crypto_alg, t->crypto_alg, @@ -411,20 +487,22 @@ static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) ip4_header_t *ih4 = (ip4_header_t *) t->packet_data; if ((ih4->ip_version_and_header_length & 0xF0) == 0x60) - s = format (s, "%U%U", format_white_space, indent, format_ip6_header, ih4); + s = + format (s, "%U%U", format_white_space, indent, format_ip6_header, ih4); else - s = format (s, "%U%U", format_white_space, indent, format_ip4_header, ih4); + s = + format (s, "%U%U", format_white_space, indent, format_ip4_header, ih4); return s; } -static uword -dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +always_inline uword +dpdk_esp_decrypt_post_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, int is_ip6) { u32 n_left_from, *from, *to_next = 0, next_index; - ipsec_sa_t * sa0; + ipsec_sa_t *sa0; u32 sa_index0 = ~0; ipsec_main_t *im = &ipsec_main; dpdk_crypto_main_t *dcm = &dpdk_crypto_main; @@ -442,14 +520,15 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, while (n_left_from > 0 && n_left_to_next > 0) { - esp_footer_t * f0; + esp_footer_t *f0; u32 bi0, iv_size, next0; - vlib_buffer_t * b0 = 0; + vlib_buffer_t *b0 = 0; ip4_header_t *ih4 = 0, *oh4 = 0; ip6_header_t *ih6 = 0, *oh6 = 0; crypto_alg_t *cipher_alg, *auth_alg; esp_header_t *esp0; u8 trunc_size, is_aead; + u16 udp_encap_adv = 0; next0 = ESP_DECRYPT_NEXT_DROP; @@ -461,7 +540,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi0); esp0 = vlib_buffer_get_current (b0); - sa_index0 = vnet_buffer(b0)->ipsec.sad_index; + sa_index0 = vnet_buffer (b0)->ipsec.sad_index; sa0 = pool_elt_at_index (im->sad, sa_index0); to_next[0] = bi0; @@ -469,13 +548,7 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, cipher_alg = vec_elt_at_index (dcm->cipher_algs, sa0->crypto_alg); auth_alg = vec_elt_at_index (dcm->auth_algs, sa0->integ_alg); -#if DPDK_NO_AEAD - is_aead = (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192 | - sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256); -#else is_aead = cipher_alg->type == RTE_CRYPTO_SYM_XFORM_AEAD; -#endif if (is_aead) auth_alg = cipher_alg; @@ -483,18 +556,22 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, iv_size = cipher_alg->iv_len; - if (sa0->use_anti_replay) + ipsec_sa_anti_replay_advance (sa0, + clib_host_to_net_u32 (esp0->seq)); + + /* if UDP encapsulation is used adjust the address of the IP header */ + if (ipsec_sa_is_set_UDP_ENCAP (sa0) + && (b0->flags & VNET_BUFFER_F_IS_IP4)) { - u32 seq; - seq = clib_host_to_net_u32(esp0->seq); - if (PREDICT_TRUE(sa0->use_esn)) - esp_replay_advance_esn(sa0, seq); - else - esp_replay_advance(sa0, seq); + udp_encap_adv = sizeof (udp_header_t); } - /* FIXME ip header */ - ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t)); + if (b0->flags & VNET_BUFFER_F_IS_IP4) + ih4 = (ip4_header_t *) + ((u8 *) esp0 - udp_encap_adv - sizeof (ip4_header_t)); + else + ih4 = (ip4_header_t *) ((u8 *) esp0 - sizeof (ip6_header_t)); + vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size); b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -505,57 +582,55 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, const u8 *padding = vlib_buffer_get_tail (b0); if (PREDICT_FALSE (memcmp (padding, pad_data, f0->pad_length))) { - clib_warning("bad padding"); + clib_warning ("bad padding"); vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, ESP_DECRYPT_ERROR_DECRYPTION_FAILED, 1); goto trace; } #endif - if (sa0->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa0)) { if (f0->next_header == IP_PROTOCOL_IP_IN_IP) next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - else if (sa0->is_tunnel_ip6 && f0->next_header == IP_PROTOCOL_IPV6) + else if (f0->next_header == IP_PROTOCOL_IPV6) next0 = ESP_DECRYPT_NEXT_IP6_INPUT; else { - clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_DECRYPTION_FAILED, - 1); + clib_warning ("next header: 0x%x", f0->next_header); + if (is_ip6) + vlib_node_increment_counter (vm, + dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, + dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); goto trace; } } - else /* transport mode */ + else /* transport mode */ { if ((ih4->ip_version_and_header_length & 0xF0) == 0x40) { u16 ih4_len = ip4_header_bytes (ih4); - vlib_buffer_advance (b0, - ih4_len); - oh4 = vlib_buffer_get_current (b0); - memmove(oh4, ih4, ih4_len); - + vlib_buffer_advance (b0, -ih4_len); next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - u16 old_ttl_prot = - ((u16) oh4->ttl) << 8 | (u16) oh4->protocol; - u16 new_ttl_prot = - ((u16) oh4->ttl) << 8 | (u16) f0->next_header; - oh4->protocol = f0->next_header; - u16 new_len = clib_host_to_net_u16 (b0->current_length); - oh4->length = new_len; - /* rfc1264 incremental checksum update */ - oh4->checksum = ~(~oh4->checksum + ~oh4->length + new_len + - ~old_ttl_prot + new_ttl_prot); + oh4 = vlib_buffer_get_current (b0); + memmove (oh4, ih4, ih4_len); + oh4->protocol = f0->next_header; + oh4->length = clib_host_to_net_u16 (b0->current_length); + oh4->checksum = ip4_header_checksum (oh4); } else if ((ih4->ip_version_and_header_length & 0xF0) == 0x60) { - /* FIXME find ip header */ - ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t)); - vlib_buffer_advance (b0, -sizeof(ip6_header_t)); + ih6 = (ip6_header_t *) ih4; + vlib_buffer_advance (b0, -sizeof (ip6_header_t)); oh6 = vlib_buffer_get_current (b0); - memmove(oh6, ih6, sizeof(ip6_header_t)); + memmove (oh6, ih6, sizeof (ip6_header_t)); next0 = ESP_DECRYPT_NEXT_IP6_INPUT; oh6->protocol = f0->next_header; @@ -564,43 +639,63 @@ dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, } else { - clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_DECRYPTION_FAILED, - 1); + clib_warning ("next header: 0x%x", f0->next_header); + if (is_ip6) + vlib_node_increment_counter (vm, + dpdk_esp6_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); + else + vlib_node_increment_counter (vm, + dpdk_esp4_decrypt_node.index, + ESP_DECRYPT_ERROR_DECRYPTION_FAILED, + 1); goto trace; } } - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0; + vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; trace: - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); + esp_decrypt_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; ih4 = vlib_buffer_get_current (b0); - clib_memcpy (tr->packet_data, ih4, sizeof (ip6_header_t)); + clib_memcpy_fast (tr->packet_data, ih4, sizeof (ip6_header_t)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, bi0, next0); + to_next, n_left_to_next, bi0, + next0); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index, - ESP_DECRYPT_POST_ERROR_PKTS, - from_frame->n_vectors); + if (is_ip6) + vlib_node_increment_counter (vm, dpdk_esp6_decrypt_post_node.index, + ESP_DECRYPT_POST_ERROR_PKTS, + from_frame->n_vectors); + else + vlib_node_increment_counter (vm, dpdk_esp4_decrypt_post_node.index, + ESP_DECRYPT_POST_ERROR_PKTS, + from_frame->n_vectors); return from_frame->n_vectors; } +VLIB_NODE_FN (dpdk_esp4_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_post_inline (vm, node, from_frame, 0 /*is_ip6 */ ); +} + /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { - .function = dpdk_esp_decrypt_post_node_fn, - .name = "dpdk-esp-decrypt-post", +VLIB_REGISTER_NODE (dpdk_esp4_decrypt_post_node) = { + .name = "dpdk-esp4-decrypt-post", .vector_size = sizeof (u32), .format_trace = format_esp_decrypt_post_trace, .type = VLIB_NODE_TYPE_INTERNAL, @@ -617,4 +712,36 @@ VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn) +VLIB_NODE_FN (dpdk_esp6_decrypt_post_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return dpdk_esp_decrypt_post_inline (vm, node, from_frame, 0 /*is_ip6 */ ); +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (dpdk_esp6_decrypt_post_node) = { + .name = "dpdk-esp6-decrypt-post", + .vector_size = sizeof (u32), + .format_trace = format_esp_decrypt_post_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings), + .error_strings = esp_decrypt_post_error_strings, + + .n_next_nodes = ESP_DECRYPT_N_NEXT, + .next_nodes = { +#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, + foreach_esp_decrypt_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */