X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fipsec%2Fipsec_if_in.c;h=63d463bd633d634034e544e1f37936eb242f8dd0;hb=1ba5bc8d8;hp=db75ab92da97e589ad3fbcbb97f8eed62ba12ef7;hpb=7cd468a3d7dee7d6c92f69a0bb7061ae208ec727;p=vpp.git diff --git a/src/vnet/ipsec/ipsec_if_in.c b/src/vnet/ipsec/ipsec_if_in.c index db75ab92da9..63d463bd633 100644 --- a/src/vnet/ipsec/ipsec_if_in.c +++ b/src/vnet/ipsec/ipsec_if_in.c @@ -21,16 +21,13 @@ #include #include - -#if DPDK_CRYPTO==1 -#define ESP_NODE "dpdk-esp-decrypt" -#else -#define ESP_NODE "esp-decrypt" -#endif +#include /* Statistics (not really errors) */ -#define foreach_ipsec_if_input_error \ -_(RX, "good packets received") +#define foreach_ipsec_if_input_error \ +_(RX, "good packets received") \ +_(DISABLED, "ipsec packets received on disabled interface") \ +_(NO_TUNNEL, "no matching tunnel") static char *ipsec_if_input_error_strings[] = { #define _(sym,string) string, @@ -46,12 +43,6 @@ typedef enum IPSEC_IF_INPUT_N_ERROR, } ipsec_if_input_error_t; -typedef enum -{ - IPSEC_IF_INPUT_NEXT_ESP_DECRYPT, - IPSEC_IF_INPUT_NEXT_DROP, - IPSEC_IF_INPUT_N_NEXT, -} ipsec_if_input_next_t; typedef struct { @@ -59,8 +50,7 @@ typedef struct u32 seq; } ipsec_if_input_trace_t; - -u8 * +static u8 * format_ipsec_if_input_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); @@ -71,101 +61,557 @@ format_ipsec_if_input_trace (u8 * s, va_list * args) return s; } -static uword -ipsec_if_input_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) + +always_inline uword +ipsec_if_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame, int is_ip6) { ipsec_main_t *im = &ipsec_main; - u32 *from, *to_next = 0, next_index; - u32 n_left_from; + vnet_main_t *vnm = im->vnet_main; + vnet_interface_main_t *vim = &vnm->interface_main; + + int is_trace = node->flags & VLIB_NODE_FLAG_TRACE; + u32 thread_index = vm->thread_index; + + u32 n_left_from, *from; + u16 nexts[VLIB_FRAME_SIZE], *next; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; - next_index = node->cached_next_index; - while (n_left_from > 0) + vlib_get_buffers (vm, from, bufs, n_left_from); + b = bufs; + next = nexts; + + clib_memset_u16 (nexts, im->esp4_decrypt_next_index, n_left_from); + + u64 n_bytes = 0, n_packets = 0; + u32 n_disabled = 0, n_no_tunnel = 0; + + u32 last_sw_if_index = ~0; + u32 last_tunnel_id = ~0; + ipsec4_tunnel_key_t last_key4; + ipsec6_tunnel_key_t last_key6; + + vlib_combined_counter_main_t *rx_counter; + vlib_combined_counter_main_t *drop_counter; + + if (is_ip6) + clib_memset (&last_key6, 0xff, sizeof (last_key6)); + else + last_key4.as_u64 = ~0; + + rx_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX; + drop_counter = vim->combined_sw_if_counters + VNET_INTERFACE_COUNTER_DROP; + + while (n_left_from >= 2) { - u32 n_left_to_next; + u32 sw_if_index0, sw_if_index1; + ip4_header_t *ip40, *ip41; + ip6_header_t *ip60, *ip61; + esp_header_t *esp0, *esp1; + u32 len0, len1; + u16 buf_adv0, buf_adv1; + u32 tid0, tid1; + ipsec_tunnel_if_t *t0, *t1; + ipsec4_tunnel_key_t key40, key41; + ipsec6_tunnel_key_t key60, key61; + + if (n_left_from >= 4) + { + CLIB_PREFETCH (b[2], CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (b[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (b[3], CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (b[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); + } + + ip40 = + (ip4_header_t *) (b[0]->data + vnet_buffer (b[0])->l3_hdr_offset); + ip41 = + (ip4_header_t *) (b[1]->data + vnet_buffer (b[1])->l3_hdr_offset); + + if (is_ip6) + { + ip60 = (ip6_header_t *) ip40; + ip61 = (ip6_header_t *) ip41; + esp0 = (esp_header_t *) ((u8 *) ip60 + sizeof (ip6_header_t)); + esp1 = (esp_header_t *) ((u8 *) ip61 + sizeof (ip6_header_t)); + buf_adv0 = sizeof (ip6_header_t); + buf_adv1 = sizeof (ip6_header_t); + } + else + { + /* NAT UDP port 4500 case, don't advance any more */ + if (ip40->protocol == IP_PROTOCOL_UDP) + { + esp0 = + (esp_header_t *) ((u8 *) ip40 + ip4_header_bytes (ip40) + + sizeof (udp_header_t)); + buf_adv0 = 0; + } + else + { + esp0 = (esp_header_t *) ((u8 *) ip40 + ip4_header_bytes (ip40)); + buf_adv0 = ip4_header_bytes (ip40); + } + /* NAT UDP port 4500 case, don't advance any more */ + if (ip41->protocol == IP_PROTOCOL_UDP) + { + esp1 = + (esp_header_t *) ((u8 *) ip41 + ip4_header_bytes (ip41) + + sizeof (udp_header_t)); + buf_adv1 = 0; + } + else + { + esp1 = (esp_header_t *) ((u8 *) ip41 + ip4_header_bytes (ip41)); + buf_adv1 = ip4_header_bytes (ip41); + } + } + + vlib_buffer_advance (b[0], buf_adv0); + vlib_buffer_advance (b[1], buf_adv1); + + len0 = vlib_buffer_length_in_chain (vm, b[0]); + len1 = vlib_buffer_length_in_chain (vm, b[1]); + + if (is_ip6) + { + key60.remote_ip = ip60->src_address; + key60.spi = esp0->spi; + + if (memcmp (&key60, &last_key6, sizeof (last_key6)) == 0) + { + tid0 = last_tunnel_id; + } + else + { + uword *p = + hash_get_mem (im->ipsec6_if_pool_index_by_key, &key60); + if (p) + { + tid0 = p[0]; + last_tunnel_id = tid0; + clib_memcpy_fast (&last_key6, &key60, sizeof (key60)); + } + else + { + n_no_tunnel++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto pkt1; + } + } + } + else /* !is_ip6 */ + { + key40.remote_ip = ip40->src_address.as_u32; + key40.spi = esp0->spi; + + if (key40.as_u64 == last_key4.as_u64) + { + tid0 = last_tunnel_id; + } + else + { + uword *p = + hash_get (im->ipsec4_if_pool_index_by_key, key40.as_u64); + if (p) + { + tid0 = p[0]; + last_tunnel_id = tid0; + last_key4.as_u64 = key40.as_u64; + } + else + { + n_no_tunnel++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto pkt1; + } + } + } + + t0 = pool_elt_at_index (im->tunnel_interfaces, tid0); + vnet_buffer (b[0])->ipsec.sad_index = t0->input_sa_index; - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + if (PREDICT_TRUE (t0->hw_if_index != ~0)) + { + vnet_buffer (b[0])->ipsec.flags = 0; + sw_if_index0 = t0->sw_if_index; + vnet_buffer (b[0])->sw_if_index[VLIB_RX] = sw_if_index0; + + if (PREDICT_FALSE (!(t0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))) + { + vlib_increment_combined_counter + (drop_counter, thread_index, sw_if_index0, 1, len0); + n_disabled++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto pkt1; + } + + if (PREDICT_TRUE (sw_if_index0 == last_sw_if_index)) + { + n_packets++; + n_bytes += len0; + } + else + { + if (n_packets) + { + vlib_increment_combined_counter + (rx_counter, thread_index, last_sw_if_index, + n_packets, n_bytes); + } + + last_sw_if_index = sw_if_index0; + n_packets = 1; + n_bytes = len0; + } + } + else + { + vnet_buffer (b[0])->ipsec.flags = IPSEC_FLAG_IPSEC_GRE_TUNNEL; + } - while (n_left_from > 0 && n_left_to_next > 0) + pkt1: + if (is_ip6) { - u32 bi0, next0; - vlib_buffer_t *b0; - ip4_header_t *ip0; - esp_header_t *esp0; - uword *p; + key61.remote_ip = ip61->src_address; + key61.spi = esp1->spi; - bi0 = to_next[0] = from[0]; - from += 1; - n_left_from -= 1; - to_next += 1; - n_left_to_next -= 1; - b0 = vlib_get_buffer (vm, bi0); - ip0 = vlib_buffer_get_current (b0); - esp0 = (esp_header_t *) ((u8 *) ip0 + ip4_header_bytes (ip0)); + if (memcmp (&key61, &last_key6, sizeof (last_key6)) == 0) + { + tid1 = last_tunnel_id; + } + else + { + uword *p = + hash_get_mem (im->ipsec6_if_pool_index_by_key, &key61); + if (p) + { + tid1 = p[0]; + last_tunnel_id = tid1; + clib_memcpy_fast (&last_key6, &key61, sizeof (key61)); + } + else + { + n_no_tunnel++; + next[1] = IPSEC_INPUT_NEXT_DROP; + goto trace1; + } + } + } + else /* !is_ip6 */ + { + key41.remote_ip = ip41->src_address.as_u32; + key41.spi = esp1->spi; - next0 = IPSEC_IF_INPUT_NEXT_DROP; + if (key41.as_u64 == last_key4.as_u64) + { + tid1 = last_tunnel_id; + } + else + { + uword *p = + hash_get (im->ipsec4_if_pool_index_by_key, key41.as_u64); + if (p) + { + tid1 = p[0]; + last_tunnel_id = tid1; + last_key4.as_u64 = key41.as_u64; + } + else + { + n_no_tunnel++; + next[1] = IPSEC_INPUT_NEXT_DROP; + goto trace1; + } + } + } - u64 key = (u64) ip0->src_address.as_u32 << 32 | - (u64) clib_net_to_host_u32 (esp0->spi); + t1 = pool_elt_at_index (im->tunnel_interfaces, tid1); + vnet_buffer (b[1])->ipsec.sad_index = t1->input_sa_index; - p = hash_get (im->ipsec_if_pool_index_by_key, key); + if (PREDICT_TRUE (t1->hw_if_index != ~0)) + { + vnet_buffer (b[1])->ipsec.flags = 0; + sw_if_index1 = t1->sw_if_index; + vnet_buffer (b[1])->sw_if_index[VLIB_RX] = sw_if_index1; - if (p) + if (PREDICT_FALSE (!(t1->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))) { - ipsec_tunnel_if_t *t; - t = pool_elt_at_index (im->tunnel_interfaces, p[0]); - vnet_buffer (b0)->ipsec.sad_index = t->input_sa_index; - vnet_buffer (b0)->ipsec.flags = - t->hw_if_index == ~0 ? IPSEC_FLAG_IPSEC_GRE_TUNNEL : 0; - vlib_buffer_advance (b0, ip4_header_bytes (ip0)); - next0 = IPSEC_IF_INPUT_NEXT_ESP_DECRYPT; + vlib_increment_combined_counter + (drop_counter, thread_index, sw_if_index1, 1, len1); + n_disabled++; + next[1] = IPSEC_INPUT_NEXT_DROP; + goto trace1; } - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + if (PREDICT_TRUE (sw_if_index1 == last_sw_if_index)) + { + n_packets++; + n_bytes += len1; + } + else + { + if (n_packets) + { + vlib_increment_combined_counter + (rx_counter, thread_index, last_sw_if_index, + n_packets, n_bytes); + } + + last_sw_if_index = sw_if_index1; + n_packets = 1; + n_bytes = len1; + } + } + else + { + vnet_buffer (b[1])->ipsec.flags = IPSEC_FLAG_IPSEC_GRE_TUNNEL; + } + + trace1: + if (PREDICT_FALSE (is_trace)) + { + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) { ipsec_if_input_trace_t *tr = - vlib_add_trace (vm, node, b0, sizeof (*tr)); + vlib_add_trace (vm, node, b[0], sizeof (*tr)); tr->spi = clib_host_to_net_u32 (esp0->spi); tr->seq = clib_host_to_net_u32 (esp0->seq); } + if (b[1]->flags & VLIB_BUFFER_IS_TRACED) + { + ipsec_if_input_trace_t *tr = + vlib_add_trace (vm, node, b[1], sizeof (*tr)); + tr->spi = clib_host_to_net_u32 (esp1->spi); + tr->seq = clib_host_to_net_u32 (esp1->seq); + } + } + + /* next */ + b += 2; + next += 2; + n_left_from -= 2; + } + while (n_left_from > 0) + { + u32 sw_if_index0; + ip4_header_t *ip40; + ip6_header_t *ip60; + esp_header_t *esp0; + u32 len0; + u16 buf_adv0; + u32 tid0; + ipsec_tunnel_if_t *t0; + ipsec4_tunnel_key_t key40; + ipsec6_tunnel_key_t key60; + + ip40 = + (ip4_header_t *) (b[0]->data + vnet_buffer (b[0])->l3_hdr_offset); + + if (is_ip6) + { + ip60 = (ip6_header_t *) ip40; + esp0 = (esp_header_t *) ((u8 *) ip60 + sizeof (ip6_header_t)); + buf_adv0 = sizeof (ip6_header_t); + } + else + { + /* NAT UDP port 4500 case, don't advance any more */ + if (ip40->protocol == IP_PROTOCOL_UDP) + { + esp0 = + (esp_header_t *) ((u8 *) ip40 + ip4_header_bytes (ip40) + + sizeof (udp_header_t)); + buf_adv0 = 0; + } + else + { + esp0 = (esp_header_t *) ((u8 *) ip40 + ip4_header_bytes (ip40)); + buf_adv0 = ip4_header_bytes (ip40); + } + } + + /* stats for the tunnel include all the data after the IP header + just like a norml IP-IP tunnel */ + vlib_buffer_advance (b[0], buf_adv0); + len0 = vlib_buffer_length_in_chain (vm, b[0]); + + if (is_ip6) + { + key60.remote_ip = ip60->src_address; + key60.spi = esp0->spi; + + if (memcmp (&key60, &last_key6, sizeof (last_key6)) == 0) + { + tid0 = last_tunnel_id; + } + else + { + uword *p = + hash_get_mem (im->ipsec6_if_pool_index_by_key, &key60); + if (p) + { + tid0 = p[0]; + last_tunnel_id = tid0; + clib_memcpy_fast (&last_key6, &key60, sizeof (key60)); + } + else + { + n_no_tunnel++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto trace00; + } + } + } + else /* !is_ip6 */ + { + key40.remote_ip = ip40->src_address.as_u32; + key40.spi = esp0->spi; - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); + if (key40.as_u64 == last_key4.as_u64) + { + tid0 = last_tunnel_id; + } + else + { + uword *p = + hash_get (im->ipsec4_if_pool_index_by_key, key40.as_u64); + if (p) + { + tid0 = p[0]; + last_tunnel_id = tid0; + last_key4.as_u64 = key40.as_u64; + } + else + { + n_no_tunnel++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto trace00; + } + } } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); + + t0 = pool_elt_at_index (im->tunnel_interfaces, tid0); + vnet_buffer (b[0])->ipsec.sad_index = t0->input_sa_index; + + if (PREDICT_TRUE (t0->hw_if_index != ~0)) + { + vnet_buffer (b[0])->ipsec.flags = 0; + sw_if_index0 = t0->sw_if_index; + vnet_buffer (b[0])->sw_if_index[VLIB_RX] = sw_if_index0; + + if (PREDICT_FALSE (!(t0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))) + { + vlib_increment_combined_counter + (drop_counter, thread_index, sw_if_index0, 1, len0); + n_disabled++; + next[0] = IPSEC_INPUT_NEXT_DROP; + goto trace00; + } + + if (PREDICT_TRUE (sw_if_index0 == last_sw_if_index)) + { + n_packets++; + n_bytes += len0; + } + else + { + if (n_packets) + { + vlib_increment_combined_counter + (rx_counter, thread_index, last_sw_if_index, + n_packets, n_bytes); + } + + last_sw_if_index = sw_if_index0; + n_packets = 1; + n_bytes = len0; + } + } + else + { + vnet_buffer (b[0])->ipsec.flags = IPSEC_FLAG_IPSEC_GRE_TUNNEL; + } + + trace00: + if (PREDICT_FALSE (is_trace)) + { + if (b[0]->flags & VLIB_BUFFER_IS_TRACED) + { + ipsec_if_input_trace_t *tr = + vlib_add_trace (vm, node, b[0], sizeof (*tr)); + tr->spi = clib_host_to_net_u32 (esp0->spi); + tr->seq = clib_host_to_net_u32 (esp0->seq); + } + } + + /* next */ + b += 1; + next += 1; + n_left_from -= 1; } - vlib_node_increment_counter (vm, ipsec_if_input_node.index, + if (n_packets) + { + vlib_increment_combined_counter (rx_counter, + thread_index, + last_sw_if_index, n_packets, n_bytes); + } + + vlib_node_increment_counter (vm, node->node_index, IPSEC_IF_INPUT_ERROR_RX, - from_frame->n_vectors); + from_frame->n_vectors - n_disabled); + vlib_node_increment_counter (vm, node->node_index, + IPSEC_IF_INPUT_ERROR_DISABLED, n_disabled); + vlib_node_increment_counter (vm, node->node_index, + IPSEC_IF_INPUT_ERROR_NO_TUNNEL, n_no_tunnel); + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, from_frame->n_vectors); return from_frame->n_vectors; } +VLIB_NODE_FN (ipsec4_if_input_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return ipsec_if_input_inline (vm, node, from_frame, 0 /* is_ip6 */ ); +} + /* *INDENT-OFF* */ -VLIB_REGISTER_NODE (ipsec_if_input_node) = { - .function = ipsec_if_input_node_fn, - .name = "ipsec-if-input", +VLIB_REGISTER_NODE (ipsec4_if_input_node) = { + .name = "ipsec4-if-input", .vector_size = sizeof (u32), .format_trace = format_ipsec_if_input_trace, .type = VLIB_NODE_TYPE_INTERNAL, - .n_errors = ARRAY_LEN(ipsec_if_input_error_strings), .error_strings = ipsec_if_input_error_strings, + .sibling_of = "ipsec4-input-feature", +}; +/* *INDENT-ON* */ - .n_next_nodes = IPSEC_IF_INPUT_N_NEXT, +VLIB_NODE_FN (ipsec6_if_input_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return ipsec_if_input_inline (vm, node, from_frame, 1 /* is_ip6 */ ); +} - .next_nodes = { - [IPSEC_IF_INPUT_NEXT_ESP_DECRYPT] = ESP_NODE, - [IPSEC_IF_INPUT_NEXT_DROP] = "error-drop", - }, +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ipsec6_if_input_node) = { + .name = "ipsec6-if-input", + .vector_size = sizeof (u32), + .format_trace = format_ipsec_if_input_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + .n_errors = ARRAY_LEN(ipsec_if_input_error_strings), + .error_strings = ipsec_if_input_error_strings, + .sibling_of = "ipsec6-input-feature", }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_input_node, ipsec_if_input_node_fn) /* * fd.io coding-style-patch-verification: ON *