From d709cbcb1ef80633af657c5427608831e5bbd919 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Tue, 26 Mar 2019 13:16:42 +0100 Subject: [PATCH] ipsec: compress ipsec_sa_t so data used by dataplane code fits in cacheline Change-Id: I81ecdf9fdcfcb017117b47dc031f93208e004d7c Signed-off-by: Damjan Marion Signed-off-by: Neale Ranns --- src/plugins/dpdk/ipsec/esp_decrypt.c | 21 +++++----- src/plugins/dpdk/ipsec/esp_encrypt.c | 23 +++++------ src/plugins/dpdk/ipsec/ipsec.c | 2 +- src/vnet/crypto/crypto.h | 14 +++++-- src/vnet/crypto/format.c | 2 +- src/vnet/ipsec/ah_decrypt.c | 11 +++--- src/vnet/ipsec/ah_encrypt.c | 16 ++++---- src/vnet/ipsec/esp.h | 11 +++--- src/vnet/ipsec/esp_decrypt.c | 13 ++++--- src/vnet/ipsec/esp_encrypt.c | 12 +++--- src/vnet/ipsec/ipsec_api.c | 16 ++++---- src/vnet/ipsec/ipsec_format.c | 13 ++++--- src/vnet/ipsec/ipsec_if.c | 10 +++-- src/vnet/ipsec/ipsec_input.c | 4 +- src/vnet/ipsec/ipsec_sa.c | 40 ++++++++++--------- src/vnet/ipsec/ipsec_sa.h | 74 +++++++++++++++++++++++++----------- 16 files changed, 164 insertions(+), 118 deletions(-) diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 20040433420..dcc276f99bc 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -234,13 +234,13 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, } /* anti-replay check */ - if (sa0->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0)) { int rv = 0; seq = clib_net_to_host_u32 (esp0->seq); - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) rv = esp_replay_check_esn (sa0, seq); else rv = esp_replay_check (sa0, seq); @@ -339,7 +339,7 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, clib_memcpy_fast (aad, esp0, 8); /* _aad[3] should always be 0 */ - if (PREDICT_FALSE (sa0->use_esn)) + if (PREDICT_FALSE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) _aad[2] = clib_host_to_net_u32 (sa0->seq_hi); else _aad[2] = 0; @@ -348,7 +348,7 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, { auth_len = sizeof (esp_header_t) + iv_size + payload_len; - if (sa0->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0)) { clib_memcpy_fast (priv->icv, digest, trunc_size); u32 *_digest = (u32 *) digest; @@ -560,18 +560,19 @@ dpdk_esp_decrypt_post_inline (vlib_main_t * vm, iv_size = cipher_alg->iv_len; - if (sa0->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0)) { u32 seq; seq = clib_host_to_net_u32 (esp0->seq); - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) esp_replay_advance_esn (sa0, seq); else esp_replay_advance (sa0, seq); } /* if UDP encapsulation is used adjust the address of the IP header */ - if (sa0->udp_encap && (b0->flags & VNET_BUFFER_F_IS_IP4)) + if (ipsec_sa_is_set_UDP_ENCAP (sa0) + && (b0->flags & VNET_BUFFER_F_IS_IP4)) { udp_encap_adv = sizeof (udp_header_t); } @@ -599,11 +600,11 @@ dpdk_esp_decrypt_post_inline (vlib_main_t * vm, goto trace; } #endif - if (sa0->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa0)) { if (f0->next_header == IP_PROTOCOL_IP_IN_IP) next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - else if (sa0->is_tunnel_ip6 + else if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0) && f0->next_header == IP_PROTOCOL_IPV6) next0 = ESP_DECRYPT_NEXT_IP6_INPUT; else @@ -629,7 +630,7 @@ dpdk_esp_decrypt_post_inline (vlib_main_t * vm, u16 ih4_len = ip4_header_bytes (ih4); vlib_buffer_advance (b0, -ih4_len - udp_encap_adv); next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - if (!sa0->udp_encap) + if (!ipsec_sa_is_set_UDP_ENCAP (sa0)) { oh4 = vlib_buffer_get_current (b0); memmove (oh4, ih4, ih4_len); diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c index d29ca55ea2e..25815d98748 100644 --- a/src/plugins/dpdk/ipsec/esp_encrypt.c +++ b/src/plugins/dpdk/ipsec/esp_encrypt.c @@ -297,13 +297,13 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, trunc_size = auth_alg->trunc_size; /* if UDP encapsulation is used adjust the address of the IP header */ - if (sa0->udp_encap && !is_ip6) + if (ipsec_sa_is_set_UDP_ENCAP (sa0) && !is_ip6) udp_encap_adv = sizeof (udp_header_t); - if (sa0->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa0)) { rewrite_len = 0; - if (!is_ip6 && !sa0->is_tunnel_ip6) /* ip4inip4 */ + if (!is_ip6 && !ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) /* ip4inip4 */ { /* in tunnel mode send it back to FIB */ priv->next = DPDK_CRYPTO_INPUT_NEXT_IP4_LOOKUP; @@ -333,7 +333,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, oh0->ip4.dst_address.as_u32 = sa0->tunnel_dst_addr.ip4.as_u32; - if (sa0->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa0)) { oh0->ip4.protocol = IP_PROTOCOL_UDP; esp0 = &ouh0->esp; @@ -343,8 +343,9 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, esp0->spi = clib_host_to_net_u32 (sa0->spi); esp0->seq = clib_host_to_net_u32 (sa0->seq); } - else if (is_ip6 && sa0->is_tunnel_ip6) /* ip6inip6 */ + else if (is_ip6 && ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) { + /* ip6inip6 */ /* in tunnel mode send it back to FIB */ priv->next = DPDK_CRYPTO_INPUT_NEXT_IP6_LOOKUP; @@ -418,7 +419,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, memmove (dst, src, rewrite_len + ip_size); oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP; esp0 = (esp_header_t *) (((u8 *) oh0) + ip_size); - if (sa0->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa0)) { oh0->ip4.protocol = IP_PROTOCOL_UDP; esp0 = (esp_header_t *) @@ -434,7 +435,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, esp0->seq = clib_host_to_net_u32 (sa0->seq); } - if (sa0->udp_encap && ouh0) + if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0) { ouh0->udp.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec); ouh0->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec); @@ -467,7 +468,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, oh0->ip4.length = clib_host_to_net_u16 (b0->current_length - rewrite_len); oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4); - if (sa0->udp_encap && ouh0) + if (ipsec_sa_is_set_UDP_ENCAP (sa0) && ouh0) { ouh0->udp.length = clib_host_to_net_u16 (clib_net_to_host_u16 @@ -512,7 +513,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, aad[1] = clib_host_to_net_u32 (sa0->seq); /* aad[3] should always be 0 */ - if (PREDICT_FALSE (sa0->use_esn)) + if (PREDICT_FALSE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) aad[2] = clib_host_to_net_u32 (sa0->seq_hi); else aad[2] = 0; @@ -521,7 +522,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, { auth_len = vlib_buffer_get_tail (b0) - ((u8 *) esp0) - trunc_size; - if (sa0->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0)) { u32 *_digest = (u32 *) digest; _digest[0] = clib_host_to_net_u32 (sa0->seq_hi); @@ -540,7 +541,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; u8 *p = vlib_buffer_get_current (b0); - if (!sa0->is_tunnel) + if (!ipsec_sa_is_set_IS_TUNNEL (sa0)) p += vnet_buffer (b0)->ip.save_rewrite_length; clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data)); } diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c index f79b4301e9c..cc06a4a87d5 100644 --- a/src/plugins/dpdk/ipsec/ipsec.c +++ b/src/plugins/dpdk/ipsec/ipsec.c @@ -258,7 +258,7 @@ crypto_set_aead_xform (struct rte_crypto_sym_xform *xform, crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb); xform->aead.iv.length = 12; xform->aead.digest_length = c->trunc_size; - xform->aead.aad_length = sa->use_esn ? 12 : 8; + xform->aead.aad_length = ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa) ? 12 : 8; xform->next = NULL; if (is_outbound) diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h index ad6286ccfcb..62db291fce2 100644 --- a/src/vnet/crypto/crypto.h +++ b/src/vnet/crypto/crypto.h @@ -57,9 +57,12 @@ typedef enum foreach_hmac_alg #undef _ VNET_CRYPTO_N_OP_TYPES, -} vnet_crypto_op_type_t; +} __attribute__ ((packed)) vnet_crypto_op_type_t; /* *INDENT-ON* */ +STATIC_ASSERT (sizeof (vnet_crypto_op_type_t) <= 2, + "crypto op type > 2 bytes"); + typedef struct { char *name; @@ -70,13 +73,16 @@ typedef enum VNET_CRYPTO_OP_STATUS_PENDING, VNET_CRYPTO_OP_STATUS_COMPLETED, VNET_CRYPTO_OP_STATUS_FAIL_NO_HANDLER, -} vnet_crypto_op_status_t; +} __attribute__ ((packed)) vnet_crypto_op_status_t; + +STATIC_ASSERT (sizeof (vnet_crypto_op_status_t) == 1, + "crypto op status > 1 byte"); typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - vnet_crypto_op_type_t op:16; - vnet_crypto_op_status_t status:8; + vnet_crypto_op_type_t op; + vnet_crypto_op_status_t status; u8 key_len, hmac_trunc_len; u16 flags; #define VNET_CRYPTO_OP_FLAG_INIT_IV 1 diff --git a/src/vnet/crypto/format.c b/src/vnet/crypto/format.c index 88c7c0fe7f1..c2786ee5afd 100644 --- a/src/vnet/crypto/format.c +++ b/src/vnet/crypto/format.c @@ -30,7 +30,7 @@ u8 * format_vnet_crypto_op (u8 * s, va_list * args) { vnet_crypto_main_t *cm = &crypto_main; - vnet_crypto_op_type_t op = va_arg (*args, vnet_crypto_op_type_t); + vnet_crypto_op_type_t op = va_arg (*args, int); // vnet_crypto_op_type_t); vnet_crypto_op_type_data_t *otd = cm->opt_data + op; return format (s, "%s-%U", otd->desc, format_vnet_crypto_alg, otd->alg); diff --git a/src/vnet/ipsec/ah_decrypt.c b/src/vnet/ipsec/ah_decrypt.c index b0916f99ef6..87e1de1b3ce 100644 --- a/src/vnet/ipsec/ah_decrypt.c +++ b/src/vnet/ipsec/ah_decrypt.c @@ -151,11 +151,11 @@ ah_decrypt_inline (vlib_main_t * vm, seq = clib_host_to_net_u32 (ah0->seq_no); /* anti-replay check */ - if (sa0->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0)) { int rv = 0; - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) rv = esp_replay_check_esn (sa0, seq); else rv = esp_replay_check (sa0, seq); @@ -210,9 +210,10 @@ ah_decrypt_inline (vlib_main_t * vm, goto trace; } - if (PREDICT_TRUE (sa0->use_anti_replay)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0))) { - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE + (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) esp_replay_advance_esn (sa0, seq); else esp_replay_advance (sa0, seq); @@ -225,7 +226,7 @@ ah_decrypt_inline (vlib_main_t * vm, icv_padding_len); i_b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; - if (PREDICT_TRUE (sa0->is_tunnel)) + if (PREDICT_TRUE (ipsec_sa_is_set_IS_TUNNEL (sa0))) { /* tunnel mode */ if (PREDICT_TRUE (ah0->nexthdr == IP_PROTOCOL_IP_IN_IP)) next0 = AH_DECRYPT_NEXT_IP4_INPUT; diff --git a/src/vnet/ipsec/ah_encrypt.c b/src/vnet/ipsec/ah_encrypt.c index 95be1412c90..2eab3ac1746 100644 --- a/src/vnet/ipsec/ah_encrypt.c +++ b/src/vnet/ipsec/ah_encrypt.c @@ -140,7 +140,7 @@ ah_encrypt_inline (vlib_main_t * vm, ttl = ih0->ip4.ttl; tos = ih0->ip4.tos; - if (PREDICT_TRUE (sa0->is_tunnel)) + if (PREDICT_TRUE (ipsec_sa_is_set_IS_TUNNEL (sa0))) { if (is_ip6) adv = -sizeof (ip6_and_ah_header_t); @@ -156,7 +156,7 @@ ah_encrypt_inline (vlib_main_t * vm, const u8 padding_len = ah_calc_icv_padding_len (icv_size, is_ip6); adv -= padding_len; /* transport mode save the eth header before it is overwritten */ - if (PREDICT_FALSE (!sa0->is_tunnel)) + if (PREDICT_FALSE (!ipsec_sa_is_set_IS_TUNNEL (sa0))) { ethernet_header_t *ieh0 = (ethernet_header_t *) ((u8 *) vlib_buffer_get_current (i_b0) - @@ -177,7 +177,7 @@ ah_encrypt_inline (vlib_main_t * vm, hop_limit = ih6_0->ip6.hop_limit; ip_version_traffic_class_and_flow_label = ih6_0->ip6.ip_version_traffic_class_and_flow_label; - if (PREDICT_TRUE (sa0->is_tunnel)) + if (PREDICT_TRUE (ipsec_sa_is_set_IS_TUNNEL (sa0))) { next_hdr_type = IP_PROTOCOL_IPV6; } @@ -206,7 +206,7 @@ ah_encrypt_inline (vlib_main_t * vm, oh0 = vlib_buffer_get_current (i_b0); clib_memset (oh0, 0, sizeof (ip4_and_ah_header_t)); - if (PREDICT_TRUE (sa0->is_tunnel)) + if (PREDICT_TRUE (ipsec_sa_is_set_IS_TUNNEL (sa0))) { next_hdr_type = IP_PROTOCOL_IP_IN_IP; } @@ -233,7 +233,8 @@ ah_encrypt_inline (vlib_main_t * vm, } - if (PREDICT_TRUE (!is_ip6 && sa0->is_tunnel && !sa0->is_tunnel_ip6)) + if (PREDICT_TRUE (!is_ip6 && ipsec_sa_is_set_IS_TUNNEL (sa0) && + !ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))) { oh0->ip4.src_address.as_u32 = sa0->tunnel_src_addr.ip4.as_u32; oh0->ip4.dst_address.as_u32 = sa0->tunnel_dst_addr.ip4.as_u32; @@ -242,7 +243,8 @@ ah_encrypt_inline (vlib_main_t * vm, vnet_buffer (i_b0)->ip.adj_index[VLIB_TX] = sa0->dpo[IPSEC_PROTOCOL_AH].dpoi_index; } - else if (is_ip6 && sa0->is_tunnel && sa0->is_tunnel_ip6) + else if (is_ip6 && ipsec_sa_is_set_IS_TUNNEL (sa0) && + ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) { oh6_0->ip6.src_address.as_u64[0] = sa0->tunnel_src_addr.ip6.as_u64[0]; @@ -282,7 +284,7 @@ ah_encrypt_inline (vlib_main_t * vm, oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4); } - if (!sa0->is_tunnel) + if (!ipsec_sa_is_set_IS_TUNNEL (sa0)) { next0 = AH_ENCRYPT_NEXT_INTERFACE_OUTPUT; vlib_buffer_advance (i_b0, -sizeof (ethernet_header_t)); diff --git a/src/vnet/ipsec/esp.h b/src/vnet/ipsec/esp.h index 063b74b7c5c..1f894ab09af 100644 --- a/src/vnet/ipsec/esp.h +++ b/src/vnet/ipsec/esp.h @@ -184,12 +184,12 @@ esp_replay_advance_esn (ipsec_sa_t * sa, u32 seq) always_inline int esp_seq_advance (ipsec_sa_t * sa) { - if (PREDICT_TRUE (sa->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa))) { if (PREDICT_FALSE (sa->seq == ESP_SEQ_MAX)) { - if (PREDICT_FALSE - (sa->use_anti_replay && sa->seq_hi == ESP_SEQ_MAX)) + if (PREDICT_FALSE (ipsec_sa_is_set_USE_ANTI_REPLAY (sa) && + sa->seq_hi == ESP_SEQ_MAX)) return 1; sa->seq_hi++; } @@ -197,7 +197,8 @@ esp_seq_advance (ipsec_sa_t * sa) } else { - if (PREDICT_FALSE (sa->use_anti_replay && sa->seq == ESP_SEQ_MAX)) + if (PREDICT_FALSE (ipsec_sa_is_set_USE_ANTI_REPLAY (sa) && + sa->seq == ESP_SEQ_MAX)) return 1; sa->seq++; } @@ -223,7 +224,7 @@ hmac_calc (vlib_main_t * vm, ipsec_sa_t * sa, u8 * data, int data_len, op->dst = signature; op->hmac_trunc_len = sa->integ_trunc_size; - if (sa->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa)) { u32 seq_hi = clib_host_to_net_u32 (sa->seq_hi); diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 7860ca33d18..7f3c320e12c 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -147,11 +147,11 @@ esp_decrypt_inline (vlib_main_t * vm, seq = clib_host_to_net_u32 (esp0->seq); /* anti-replay check */ - if (sa0->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0)) { int rv = 0; - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) rv = esp_replay_check_esn (sa0, seq); else rv = esp_replay_check (sa0, seq); @@ -197,9 +197,9 @@ esp_decrypt_inline (vlib_main_t * vm, } } - if (PREDICT_TRUE (sa0->use_anti_replay)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_ANTI_REPLAY (sa0))) { - if (PREDICT_TRUE (sa0->use_esn)) + if (PREDICT_TRUE (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0))) esp_replay_advance_esn (sa0, seq); else esp_replay_advance (sa0, seq); @@ -222,7 +222,8 @@ esp_decrypt_inline (vlib_main_t * vm, ob[0]->current_data = sizeof (ethernet_header_t); /* transport mode */ - if (PREDICT_FALSE (!sa0->is_tunnel && !sa0->is_tunnel_ip6)) + if (PREDICT_FALSE (!ipsec_sa_is_set_IS_TUNNEL (sa0) && + !ipsec_sa_is_set_IS_TUNNEL_V6 (sa0))) { tunnel_mode = 0; @@ -235,7 +236,7 @@ esp_decrypt_inline (vlib_main_t * vm, else { ip_hdr_size = sizeof (ip4_header_t); - if (sa0->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa0)) ih4 = (ip4_header_t *) ((u8 *) esp0 - ip_hdr_size - sizeof (udp_header_t)); else diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index c792a149f12..6e5007151b9 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -289,7 +289,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* space for IV */ hdr_len = iv_sz; - if (sa0->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa0)) { payload = vlib_buffer_get_current (b[0]); next_hdr_ptr = esp_add_footer_and_icv (b[0], block_sz, icv_sz); @@ -303,7 +303,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, esp = (esp_header_t *) (payload - hdr_len); /* optional UDP header */ - if (sa0->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa0)) { hdr_len += sizeof (udp_header_t); esp_fill_udp_hdr (sa0, (udp_header_t *) (payload - hdr_len), @@ -311,7 +311,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } /* IP header */ - if (sa0->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) { ip6_header_t *ip6; u16 len = sizeof (ip6_header_t); @@ -361,7 +361,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, esp = (esp_header_t *) (payload - hdr_len); /* optional UDP header */ - if (sa0->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa0)) { hdr_len += sizeof (udp_header_t); udp = (udp_header_t *) (payload - hdr_len); @@ -434,7 +434,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, op->len = payload_len - icv_sz + iv_sz + sizeof (esp_header_t); op->flags = 0; op->user_data = b - bufs; - if (sa0->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa0)) { u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi); clib_memcpy_fast (op->dst, &seq_hi, sizeof (seq_hi)); @@ -455,7 +455,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tr->sa_index = sa_index0; tr->spi = sa0->spi; tr->seq = sa0->seq - 1; - tr->udp_encap = sa0->udp_encap; + tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa0); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; } diff --git a/src/vnet/ipsec/ipsec_api.c b/src/vnet/ipsec/ipsec_api.c index f43cae99cc9..da175b2e5b9 100644 --- a/src/vnet/ipsec/ipsec_api.c +++ b/src/vnet/ipsec/ipsec_api.c @@ -320,15 +320,15 @@ ipsec_sad_flags_encode (const ipsec_sa_t * sa) { vl_api_ipsec_sad_flags_t flags = IPSEC_API_SAD_FLAG_NONE; - if (sa->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa)) flags |= IPSEC_API_SAD_FLAG_USE_EXTENDED_SEQ_NUM; - if (sa->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa)) flags |= IPSEC_API_SAD_FLAG_USE_ANTI_REPLAY; - if (sa->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa)) flags |= IPSEC_API_SAD_FLAG_IS_TUNNEL; - if (sa->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) flags |= IPSEC_API_SAD_FLAG_IS_TUNNEL_V6; - if (sa->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa)) flags |= IPSEC_API_SAD_FLAG_UDP_ENCAP; return clib_host_to_net_u32 (flags); @@ -690,7 +690,7 @@ send_ipsec_sa_details (ipsec_sa_t * sa, vl_api_registration_t * reg, mp->entry.flags = ipsec_sad_flags_encode (sa); - if (sa->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa)) { ip_address_encode (&sa->tunnel_src_addr, IP46_TYPE_ANY, &mp->entry.tunnel_src); @@ -702,12 +702,12 @@ send_ipsec_sa_details (ipsec_sa_t * sa, vl_api_registration_t * reg, mp->salt = clib_host_to_net_u32 (sa->salt); mp->seq_outbound = clib_host_to_net_u64 (((u64) sa->seq)); mp->last_seq_inbound = clib_host_to_net_u64 (((u64) sa->last_seq)); - if (sa->use_esn) + if (ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa)) { mp->seq_outbound |= (u64) (clib_host_to_net_u32 (sa->seq_hi)); mp->last_seq_inbound |= (u64) (clib_host_to_net_u32 (sa->last_seq_hi)); } - if (sa->use_anti_replay) + if (ipsec_sa_is_set_USE_ANTI_REPLAY (sa)) mp->replay_window = clib_host_to_net_u64 (sa->replay_window); vl_api_send_msg (reg, (u8 *) mp); diff --git a/src/vnet/ipsec/ipsec_format.c b/src/vnet/ipsec/ipsec_format.c index e682386eee1..1ad3a53c45b 100644 --- a/src/vnet/ipsec/ipsec_format.c +++ b/src/vnet/ipsec/ipsec_format.c @@ -256,12 +256,13 @@ format_ipsec_sa (u8 * s, va_list * args) s = format (s, "[%d] sa 0x%x spi %u mode %s%s protocol %s%s%s%s", sai, sa->id, sa->spi, - sa->is_tunnel ? "tunnel" : "transport", - sa->is_tunnel_ip6 ? "-ip6" : "", + ipsec_sa_is_set_IS_TUNNEL (sa) ? "tunnel" : "transport", + ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? "-ip6" : "", sa->protocol ? "esp" : "ah", - sa->udp_encap ? " udp-encap-enabled" : "", - sa->use_anti_replay ? " anti-replay" : "", - sa->use_esn ? " extended-sequence-number" : ""); + ipsec_sa_is_set_UDP_ENCAP (sa) ? " udp-encap-enabled" : "", + ipsec_sa_is_set_USE_ANTI_REPLAY (sa) ? " anti-replay" : "", + ipsec_sa_is_set_USE_EXTENDED_SEQ_NUM (sa) ? + " extended-sequence-number" : ""); s = format (s, "\n seq %u seq-hi %u", sa->seq, sa->seq_hi); s = format (s, "\n last-seq %u last-seq-hi %u window %U", sa->last_seq, sa->last_seq_hi, @@ -276,7 +277,7 @@ format_ipsec_sa (u8 * s, va_list * args) vlib_get_combined_counter (&ipsec_sa_counters, sai, &counts); s = format (s, "\n packets %u bytes %u", counts.packets, counts.bytes); - if (sa->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa)) { tx_table_id = fib_table_get_table_id (sa->tx_fib_index, FIB_PROTOCOL_IP4); diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c index bf3460423fb..b1c05346dd0 100644 --- a/src/vnet/ipsec/ipsec_if.c +++ b/src/vnet/ipsec/ipsec_if.c @@ -466,7 +466,7 @@ ipsec_add_del_ipsec_gre_tunnel (vnet_main_t * vnm, osa = p[0]; sa = pool_elt_at_index (im->sad, p[0]); - if (sa->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa)) { key.remote_ip = sa->tunnel_dst_addr.ip4.as_u32; key.spi = clib_host_to_net_u32 (sa->spi); @@ -599,13 +599,14 @@ ipsec_set_interface_sa (vnet_main_t * vnm, u32 hw_if_index, u32 sa_id, old_sa_index = t->input_sa_index; old_sa = pool_elt_at_index (im->sad, old_sa_index); - if (sa->is_tunnel_ip6 ^ old_sa->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ^ + ipsec_sa_is_set_IS_TUNNEL_V6 (old_sa)) { clib_warning ("IPsec interface SA endpoints type can't be changed"); return VNET_API_ERROR_INVALID_VALUE; } - if (sa->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) { ipsec6_tunnel_key_t key; @@ -651,7 +652,8 @@ ipsec_set_interface_sa (vnet_main_t * vnm, u32 hw_if_index, u32 sa_id, old_sa_index = t->output_sa_index; old_sa = pool_elt_at_index (im->sad, old_sa_index); - if (sa->is_tunnel_ip6 ^ old_sa->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ^ + ipsec_sa_is_set_IS_TUNNEL_V6 (old_sa)) { clib_warning ("IPsec interface SA endpoints type can't be changed"); return VNET_API_ERROR_INVALID_VALUE; diff --git a/src/vnet/ipsec/ipsec_input.c b/src/vnet/ipsec/ipsec_input.c index 0070e33e729..970211eccaa 100644 --- a/src/vnet/ipsec/ipsec_input.c +++ b/src/vnet/ipsec/ipsec_input.c @@ -84,7 +84,7 @@ ipsec_input_protect_policy_match (ipsec_spd_t * spd, u32 sa, u32 da, u32 spi) if (spi != s->spi) continue; - if (s->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (s)) { if (da != clib_net_to_host_u32 (s->tunnel_dst_addr.ip4.as_u32)) continue; @@ -140,7 +140,7 @@ ipsec6_input_protect_policy_match (ipsec_spd_t * spd, if (spi != s->spi) continue; - if (s->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (s)) { if (!ip6_address_is_equal (sa, &s->tunnel_src_addr.ip6)) continue; diff --git a/src/vnet/ipsec/ipsec_sa.c b/src/vnet/ipsec/ipsec_sa.c index 3e29c0406e3..3d62395bd7c 100644 --- a/src/vnet/ipsec/ipsec_sa.c +++ b/src/vnet/ipsec/ipsec_sa.c @@ -73,17 +73,18 @@ ipsec_sa_stack (ipsec_sa_t * sa) fib_forward_chain_type_t fct; dpo_id_t tmp = DPO_INVALID; - fct = fib_forw_chain_type_from_fib_proto ((sa->is_tunnel_ip6 ? - FIB_PROTOCOL_IP6 : - FIB_PROTOCOL_IP4)); + fct = + fib_forw_chain_type_from_fib_proto ((ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? + FIB_PROTOCOL_IP6 : + FIB_PROTOCOL_IP4)); fib_entry_contribute_forwarding (sa->fib_entry_index, fct, &tmp); - dpo_stack_from_node ((sa->is_tunnel_ip6 ? + dpo_stack_from_node ((ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? im->ah6_encrypt_node_index : im->ah4_encrypt_node_index), &sa->dpo[IPSEC_PROTOCOL_AH], &tmp); - dpo_stack_from_node ((sa->is_tunnel_ip6 ? + dpo_stack_from_node ((ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? im->esp6_encrypt_node_index : im->esp4_encrypt_node_index), &sa->dpo[IPSEC_PROTOCOL_ESP], &tmp); @@ -135,7 +136,7 @@ ipsec_sa_add (u32 id, if (p) return VNET_API_ERROR_ENTRY_ALREADY_EXISTS; - pool_get_zero (im->sad, sa); + pool_get_aligned_zero (im->sad, sa, CLIB_CACHE_LINE_BYTES); fib_node_init (&sa->node, FIB_NODE_TYPE_IPSEC_SA); sa_index = sa - im->sad; @@ -155,15 +156,15 @@ ipsec_sa_add (u32 id, ip46_address_copy (&sa->tunnel_dst_addr, tun_dst); if (flags & IPSEC_SA_FLAG_USE_EXTENDED_SEQ_NUM) - sa->use_esn = 1; + ipsec_sa_set_USE_EXTENDED_SEQ_NUM (sa); if (flags & IPSEC_SA_FLAG_USE_ANTI_REPLAY) - sa->use_anti_replay = 1; + ipsec_sa_set_USE_ANTI_REPLAY (sa); if (flags & IPSEC_SA_FLAG_IS_TUNNEL) - sa->is_tunnel = 1; + ipsec_sa_set_IS_TUNNEL (sa); if (flags & IPSEC_SA_FLAG_IS_TUNNEL_V6) - sa->is_tunnel_ip6 = 1; + ipsec_sa_set_IS_TUNNEL_V6 (sa); if (flags & IPSEC_SA_FLAG_UDP_ENCAP) - sa->udp_encap = 1; + ipsec_sa_set_UDP_ENCAP (sa); err = ipsec_check_support_cb (im, sa); if (err) @@ -180,13 +181,13 @@ ipsec_sa_add (u32 id, return VNET_API_ERROR_SYSCALL_ERROR_1; } - if (sa->is_tunnel) + if (ipsec_sa_is_set_IS_TUNNEL (sa)) { - fib_protocol_t fproto = (sa->is_tunnel_ip6 ? + fib_protocol_t fproto = (ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4); fib_prefix_t pfx = { .fp_addr = sa->tunnel_dst_addr, - .fp_len = (sa->is_tunnel_ip6 ? 128 : 32), + .fp_len = (ipsec_sa_is_set_IS_TUNNEL_V6 (sa) ? 128 : 32), .fp_proto = fproto, }; sa->tx_fib_index = fib_table_find (fproto, tx_table_id); @@ -205,7 +206,7 @@ ipsec_sa_add (u32 id, ipsec_sa_stack (sa); /* generate header templates */ - if (sa->is_tunnel_ip6) + if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa)) { sa->ip6_hdr.ip_version_traffic_class_and_flow_label = 0x60; sa->ip6_hdr.hop_limit = 254; @@ -217,7 +218,7 @@ ipsec_sa_add (u32 id, sa->tunnel_dst_addr.ip6.as_u64[0]; sa->ip6_hdr.dst_address.as_u64[1] = sa->tunnel_dst_addr.ip6.as_u64[1]; - if (sa->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa)) sa->ip6_hdr.protocol = IP_PROTOCOL_UDP; else sa->ip6_hdr.protocol = IP_PROTOCOL_IPSEC_ESP; @@ -229,7 +230,7 @@ ipsec_sa_add (u32 id, sa->ip4_hdr.src_address.as_u32 = sa->tunnel_src_addr.ip4.as_u32; sa->ip4_hdr.dst_address.as_u32 = sa->tunnel_dst_addr.ip4.as_u32; - if (sa->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa)) sa->ip4_hdr.protocol = IP_PROTOCOL_UDP; else sa->ip4_hdr.protocol = IP_PROTOCOL_IPSEC_ESP; @@ -237,7 +238,7 @@ ipsec_sa_add (u32 id, } } - if (sa->udp_encap) + if (ipsec_sa_is_set_UDP_ENCAP (sa)) { sa->udp_hdr.src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec); sa->udp_hdr.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipsec); @@ -277,7 +278,8 @@ ipsec_sa_del (u32 id) err = ipsec_call_add_del_callbacks (im, sa, sa_index, 0); if (err) return VNET_API_ERROR_SYSCALL_ERROR_1; - if (sa->is_tunnel) + + if (ipsec_sa_is_set_IS_TUNNEL (sa)) { fib_entry_child_remove (sa->fib_entry_index, sa->sibling); fib_table_entry_special_remove diff --git a/src/vnet/ipsec/ipsec_sa.h b/src/vnet/ipsec/ipsec_sa.h index d3335a2d956..1cd215340f5 100644 --- a/src/vnet/ipsec/ipsec_sa.h +++ b/src/vnet/ipsec/ipsec_sa.h @@ -94,55 +94,83 @@ typedef enum ipsec_sad_flags_t_ #define _(v, f, s) IPSEC_SA_FLAG_##f = v, foreach_ipsec_sa_flags #undef _ -} ipsec_sa_flags_t; +} __attribute__ ((packed)) ipsec_sa_flags_t; + +STATIC_ASSERT (sizeof (ipsec_sa_flags_t) == 1, "IPSEC SA flags > 1 byte"); typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + + /* flags */ + ipsec_sa_flags_t flags; + + u8 crypto_iv_size; + u8 crypto_block_size; + u8 integ_trunc_size; + u32 spi; + u32 seq; + u32 seq_hi; + u32 last_seq; + u32 last_seq_hi; + u64 replay_window; + + vnet_crypto_op_type_t crypto_enc_op_type; + vnet_crypto_op_type_t crypto_dec_op_type; + vnet_crypto_op_type_t integ_op_type; + + dpo_id_t dpo[IPSEC_N_PROTOCOLS]; + + /* data accessed by dataplane code should be above this comment */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); + + union + { + ip4_header_t ip4_hdr; + ip6_header_t ip6_hdr; + }; + udp_header_t udp_hdr; + + fib_node_t node; u32 id; - u32 spi; u32 stat_index; ipsec_protocol_t protocol; ipsec_crypto_alg_t crypto_alg; ipsec_key_t crypto_key; - u8 crypto_iv_size; - u8 crypto_block_size; - vnet_crypto_op_type_t crypto_enc_op_type; - vnet_crypto_op_type_t crypto_dec_op_type; ipsec_integ_alg_t integ_alg; ipsec_key_t integ_key; - vnet_crypto_op_type_t integ_op_type; - u8 integ_trunc_size; - u8 use_esn; - u8 use_anti_replay; - - u8 is_tunnel; - u8 is_tunnel_ip6; - u8 udp_encap; ip46_address_t tunnel_src_addr; ip46_address_t tunnel_dst_addr; - ip4_header_t ip4_hdr; - ip6_header_t ip6_hdr; - udp_header_t udp_hdr; fib_node_index_t fib_entry_index; u32 sibling; - dpo_id_t dpo[IPSEC_N_PROTOCOLS]; u32 tx_fib_index; u32 salt; /* runtime */ - u32 seq; - u32 seq_hi; - u32 last_seq; - u32 last_seq_hi; - u64 replay_window; } ipsec_sa_t; +STATIC_ASSERT_OFFSET_OF (ipsec_sa_t, cacheline1, CLIB_CACHE_LINE_BYTES); + +#define _(a,v,s) \ + always_inline int \ + ipsec_sa_is_set_##v (const ipsec_sa_t *sa) { \ + return (sa->flags & IPSEC_SA_FLAG_##v); \ + } +foreach_ipsec_sa_flags +#undef _ +#define _(a,v,s) \ + always_inline int \ + ipsec_sa_set_##v (ipsec_sa_t *sa) { \ + return (sa->flags |= IPSEC_SA_FLAG_##v); \ + } + foreach_ipsec_sa_flags +#undef _ /** * @brief * SA packet & bytes counters -- 2.16.6