X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fip%2Freass%2Fip4_sv_reass.c;h=31177809f7bc400d419f65f29caeeda8ccdf855f;hb=5a58e7c546239da5c500611d60cf4e7635f138c5;hp=a926f2adf10f550707dee911433541af0fa71e83;hpb=407f593dc0d06453f0b5c124b3a48edd52da6704;p=vpp.git diff --git a/src/vnet/ip/reass/ip4_sv_reass.c b/src/vnet/ip/reass/ip4_sv_reass.c index a926f2adf10..31177809f7b 100644 --- a/src/vnet/ip/reass/ip4_sv_reass.c +++ b/src/vnet/ip/reass/ip4_sv_reass.c @@ -48,7 +48,7 @@ typedef struct { struct { - u32 xx_id; + u32 fib_index; ip4_address_t src; ip4_address_t dst; u16 frag_id; @@ -98,6 +98,9 @@ typedef struct bool is_complete; // ip protocol u8 ip_proto; + u8 icmp_type_or_tcp_flags; + u32 tcp_ack_number; + u32 tcp_seq_number; // l4 src port u16 l4_src_port; // l4 dst port @@ -151,6 +154,9 @@ typedef struct // reference count for enabling/disabling feature - per interface u32 *feature_use_refcount_per_intf; + // reference count for enabling/disabling feature - per interface + u32 *output_feature_use_refcount_per_intf; + } ip4_sv_reass_main_t; extern ip4_sv_reass_main_t ip4_sv_reass_main; @@ -172,6 +178,7 @@ typedef enum REASS_FRAGMENT_CACHE, REASS_FINISH, REASS_FRAGMENT_FORWARD, + REASS_PASSTHROUGH, } ip4_sv_reass_trace_operation_e; typedef struct @@ -182,6 +189,7 @@ typedef struct u8 ip_proto; u16 l4_src_port; u16 l4_dst_port; + int l4_layer_truncated; } ip4_sv_reass_trace_t; extern vlib_node_registration_t ip4_sv_reass_node; @@ -193,7 +201,10 @@ format_ip4_sv_reass_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); ip4_sv_reass_trace_t *t = va_arg (*args, ip4_sv_reass_trace_t *); - s = format (s, "reass id: %u, op id: %u ", t->reass_id, t->op_id); + if (REASS_PASSTHROUGH != t->action) + { + s = format (s, "reass id: %u, op id: %u ", t->reass_id, t->op_id); + } switch (t->action) { case REASS_FRAGMENT_CACHE: @@ -211,25 +222,44 @@ format_ip4_sv_reass_trace (u8 * s, va_list * args) t->ip_proto, clib_net_to_host_u16 (t->l4_src_port), clib_net_to_host_u16 (t->l4_dst_port)); break; + case REASS_PASSTHROUGH: + s = format (s, "[not-fragmented]"); + break; + } + if (t->l4_layer_truncated) + { + s = format (s, " [l4-layer-truncated]"); } return s; } static void -ip4_sv_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node, - ip4_sv_reass_main_t * rm, ip4_sv_reass_t * reass, - u32 bi, ip4_sv_reass_trace_operation_e action, - u32 ip_proto, u16 l4_src_port, u16 l4_dst_port) +ip4_sv_reass_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, + ip4_sv_reass_t *reass, u32 bi, + ip4_sv_reass_trace_operation_e action, u32 ip_proto, + u16 l4_src_port, u16 l4_dst_port, + int l4_layer_truncated) { vlib_buffer_t *b = vlib_get_buffer (vm, bi); + if (pool_is_free_index + (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b))) + { + // this buffer's trace is gone + b->flags &= ~VLIB_BUFFER_IS_TRACED; + return; + } ip4_sv_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0])); - t->reass_id = reass->id; + if (reass) + { + t->reass_id = reass->id; + t->op_id = reass->trace_op_counter; + ++reass->trace_op_counter; + } t->action = action; - t->op_id = reass->trace_op_counter; t->ip_proto = ip_proto; t->l4_src_port = l4_src_port; t->l4_dst_port = l4_dst_port; - ++reass->trace_op_counter; + t->l4_layer_truncated = l4_layer_truncated; #if 0 static u8 *s = NULL; s = format (s, "%U", format_ip4_sv_reass_trace, NULL, NULL, t); @@ -289,10 +319,9 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, ip4_sv_reass_kv_t * kv, u8 * do_handoff) { ip4_sv_reass_t *reass = NULL; - f64 now = vlib_time_now (rm->vlib_main); + f64 now = vlib_time_now (vm); - if (!clib_bihash_search_16_8 - (&rm->hash, (clib_bihash_kv_16_8_t *) kv, (clib_bihash_kv_16_8_t *) kv)) + if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv)) { if (vm->thread_index != kv->v.thread_index) { @@ -316,7 +345,7 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, if (rt->reass_n >= rm->max_reass_n && rm->max_reass_n) { - reass = pool_elt_at_index (rt->pool, rt->lru_last); + reass = pool_elt_at_index (rt->pool, rt->lru_first); ip4_sv_reass_free (vm, rm, rt, reass); } @@ -340,13 +369,13 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, rt->lru_first = rt->lru_last = reass - rt->pool; } - reass->key.as_u64[0] = ((clib_bihash_kv_16_8_t *) kv)->key[0]; - reass->key.as_u64[1] = ((clib_bihash_kv_16_8_t *) kv)->key[1]; + reass->key.as_u64[0] = kv->kv.key[0]; + reass->key.as_u64[1] = kv->kv.key[1]; kv->v.reass_index = (reass - rt->pool); kv->v.thread_index = vm->thread_index; reass->last_heard = now; - if (clib_bihash_add_del_16_8 (&rm->hash, (clib_bihash_kv_16_8_t *) kv, 1)) + if (clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 1)) { ip4_sv_reass_free (vm, rm, rt, reass); reass = NULL; @@ -356,37 +385,49 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, } always_inline ip4_sv_reass_rc_t -ip4_sv_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node, - ip4_sv_reass_main_t * rm, ip4_sv_reass_per_thread_t * rt, - ip4_sv_reass_t * reass, u32 bi0) +ip4_sv_reass_update (vlib_main_t *vm, vlib_node_runtime_t *node, + ip4_sv_reass_main_t *rm, ip4_header_t *ip0, + ip4_sv_reass_t *reass, u32 bi0) { - vlib_buffer_t *fb = vlib_get_buffer (vm, bi0); + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); ip4_sv_reass_rc_t rc = IP4_SV_REASS_RC_OK; - ip4_header_t *fip = vlib_buffer_get_current (fb); - const u32 fragment_first = ip4_get_fragment_offset_bytes (fip); + const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0); if (0 == fragment_first) { - reass->ip_proto = fip->protocol; - reass->l4_src_port = ip4_get_port (fip, 1); - reass->l4_dst_port = ip4_get_port (fip, 0); + reass->ip_proto = ip0->protocol; + reass->l4_src_port = ip4_get_port (ip0, 1); + reass->l4_dst_port = ip4_get_port (ip0, 0); if (!reass->l4_src_port || !reass->l4_dst_port) return IP4_SV_REASS_RC_UNSUPP_IP_PROTO; + if (IP_PROTOCOL_TCP == reass->ip_proto) + { + reass->icmp_type_or_tcp_flags = ((tcp_header_t *) (ip0 + 1))->flags; + reass->tcp_ack_number = ((tcp_header_t *) (ip0 + 1))->ack_number; + reass->tcp_seq_number = ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == reass->ip_proto) + { + reass->icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } reass->is_complete = true; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, REASS_FINISH, - reass->ip_proto, reass->l4_src_port, - reass->l4_dst_port); + ip4_sv_reass_add_trace ( + vm, node, reass, bi0, REASS_FINISH, reass->ip_proto, + reass->l4_src_port, reass->l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); } } vec_add1 (reass->cached_buffers, bi0); if (!reass->is_complete) { - if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED)) + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, - REASS_FRAGMENT_CACHE, ~0, ~0, ~0); + ip4_sv_reass_add_trace ( + vm, node, reass, bi0, REASS_FRAGMENT_CACHE, ~0, ~0, ~0, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); } if (vec_len (reass->cached_buffers) > rm->max_reass_len) { @@ -396,9 +437,23 @@ ip4_sv_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node, return rc; } +always_inline int +l4_layer_truncated (ip4_header_t *ip) +{ + static const int l4_layer_length[256] = { + [IP_PROTOCOL_TCP] = sizeof (tcp_header_t), + [IP_PROTOCOL_UDP] = sizeof (udp_header_t), + [IP_PROTOCOL_ICMP] = sizeof (icmp46_header_t), + }; + + return ((u8 *) ip + ip4_header_bytes (ip) + l4_layer_length[ip->protocol] > + (u8 *) ip + clib_net_to_host_u16 (ip->length)); +} + always_inline uword ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, bool is_feature, bool is_custom) + vlib_frame_t * frame, bool is_feature, + bool is_output_feature, bool is_custom) { u32 *from = vlib_frame_vector_args (frame); u32 n_left_from, n_left_to_next, *to_next, next_index; @@ -409,6 +464,243 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, n_left_from = frame->n_vectors; next_index = node->cached_next_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + vlib_get_buffers (vm, from, bufs, n_left_from); + u16 nexts[VLIB_FRAME_SIZE], *next = nexts; + b = bufs; + + /* optimistic case first - no fragments */ + while (n_left_from >= 2) + { + vlib_buffer_t *b0, *b1; + u32 next0, next1; + b0 = *b; + b++; + b1 = *b; + b++; + + /* Prefetch next iteration. */ + if (PREDICT_TRUE (n_left_from >= 4)) + { + vlib_buffer_t *p2, *p3; + + p2 = *b; + p3 = *(b + 1); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); + } + + ip4_header_t *ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); + ip4_header_t *ip1 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b1), + (is_output_feature ? 1 : 0) * + vnet_buffer (b1)-> + ip.save_rewrite_length); + + if (PREDICT_FALSE + (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0)) + || (ip4_get_fragment_more (ip1) || ip4_get_fragment_offset (ip1))) + { + // fragment found, go slow path + b -= 2; + if (b - bufs > 0) + { + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + b - bufs); + } + goto slow_path; + } + if (is_feature) + { + vnet_feature_next (&next0, b0); + } + else + { + next0 = is_custom ? vnet_buffer (b0)->ip.reass.next_index : + IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol; + if (l4_layer_truncated (ip0)) + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 1; + vnet_buffer (b0)->ip.reass.l4_src_port = 0; + vnet_buffer (b0)->ip.reass.l4_dst_port = 0; + } + else + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 0; + if (IP_PROTOCOL_TCP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip0 + 1))->flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip0 + 1))->ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } + vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1); + vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0); + } + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace ( + vm, node, NULL, from[(b - 2) - bufs], REASS_PASSTHROUGH, + vnet_buffer (b0)->ip.reass.ip_proto, + vnet_buffer (b0)->ip.reass.l4_src_port, + vnet_buffer (b0)->ip.reass.l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); + } + if (is_feature) + { + vnet_feature_next (&next1, b1); + } + else + { + next1 = is_custom ? vnet_buffer (b1)->ip.reass.next_index : + IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b1)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b1)->ip.reass.ip_proto = ip1->protocol; + if (l4_layer_truncated (ip1)) + { + vnet_buffer (b1)->ip.reass.l4_layer_truncated = 1; + vnet_buffer (b1)->ip.reass.l4_src_port = 0; + vnet_buffer (b1)->ip.reass.l4_dst_port = 0; + } + else + { + vnet_buffer (b1)->ip.reass.l4_layer_truncated = 0; + if (IP_PROTOCOL_TCP == ip1->protocol) + { + vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip1 + 1))->flags; + vnet_buffer (b1)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip1 + 1))->ack_number; + vnet_buffer (b1)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip1 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip1->protocol) + { + vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip1 + 1))->type; + } + vnet_buffer (b1)->ip.reass.l4_src_port = ip4_get_port (ip1, 1); + vnet_buffer (b1)->ip.reass.l4_dst_port = ip4_get_port (ip1, 0); + } + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace ( + vm, node, NULL, from[(b - 1) - bufs], REASS_PASSTHROUGH, + vnet_buffer (b1)->ip.reass.ip_proto, + vnet_buffer (b1)->ip.reass.l4_src_port, + vnet_buffer (b1)->ip.reass.l4_dst_port, + vnet_buffer (b1)->ip.reass.l4_layer_truncated); + } + + n_left_from -= 2; + next[0] = next0; + next[1] = next1; + next += 2; + } + + while (n_left_from > 0) + { + vlib_buffer_t *b0; + u32 next0; + b0 = *b; + b++; + + ip4_header_t *ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); + if (PREDICT_FALSE + (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0))) + { + // fragment found, go slow path + b -= 1; + if (b - bufs > 0) + { + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + b - bufs); + } + goto slow_path; + } + if (is_feature) + { + vnet_feature_next (&next0, b0); + } + else + { + next0 = + is_custom ? vnet_buffer (b0)->ip. + reass.next_index : IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol; + if (l4_layer_truncated (ip0)) + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 1; + } + else + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 0; + if (IP_PROTOCOL_TCP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip0 + 1))->flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip0 + 1))->ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } + vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1); + vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0); + } + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace ( + vm, node, NULL, from[(b - 1) - bufs], REASS_PASSTHROUGH, + vnet_buffer (b0)->ip.reass.ip_proto, + vnet_buffer (b0)->ip.reass.l4_src_port, + vnet_buffer (b0)->ip.reass.l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); + } + + n_left_from -= 1; + next[0] = next0; + next += 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + frame->n_vectors); + + goto done; + +slow_path: + + from += b - bufs; + while (n_left_from > 0) { vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); @@ -423,14 +715,61 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, bi0 = from[0]; b0 = vlib_get_buffer (vm, bi0); - ip4_header_t *ip0 = vlib_buffer_get_current (b0); + ip4_header_t *ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0)) { // this is a regular packet - no fragmentation + if (is_custom) + { + next0 = vnet_buffer (b0)->ip.reass.next_index; + } + else + { + next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0; vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol; - vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1); - vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0); - next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; + if (l4_layer_truncated (ip0)) + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 1; + vnet_buffer (b0)->ip.reass.l4_src_port = 0; + vnet_buffer (b0)->ip.reass.l4_dst_port = 0; + } + else + { + vnet_buffer (b0)->ip.reass.l4_layer_truncated = 0; + if (IP_PROTOCOL_TCP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip0 + 1))->flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip0 + 1))->ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } + vnet_buffer (b0)->ip.reass.l4_src_port = + ip4_get_port (ip0, 1); + vnet_buffer (b0)->ip.reass.l4_dst_port = + ip4_get_port (ip0, 0); + } + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace ( + vm, node, NULL, bi0, REASS_PASSTHROUGH, + vnet_buffer (b0)->ip.reass.ip_proto, + vnet_buffer (b0)->ip.reass.l4_src_port, + vnet_buffer (b0)->ip.reass.l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); + } goto packet_enqueue; } const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0); @@ -441,6 +780,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { next0 = IP4_SV_REASSEMBLY_NEXT_DROP; error0 = IP4_ERROR_REASS_MALFORMED_PACKET; + b0->error = node->errors[error0]; goto packet_enqueue; } ip4_sv_reass_kv_t kv; @@ -469,29 +809,43 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { next0 = IP4_SV_REASSEMBLY_NEXT_DROP; error0 = IP4_ERROR_REASS_LIMIT_REACHED; + b0->error = node->errors[error0]; goto packet_enqueue; } if (reass->is_complete) { + if (is_custom) + { + next0 = vnet_buffer (b0)->ip.reass.next_index; + } + else + { + next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = + ! !fragment_first; vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto; + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + reass->icmp_type_or_tcp_flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + reass->tcp_ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + reass->tcp_seq_number; vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port; vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port; - next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; - error0 = IP4_ERROR_NONE; if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, - REASS_FRAGMENT_FORWARD, - reass->ip_proto, - reass->l4_src_port, - reass->l4_dst_port); + ip4_sv_reass_add_trace ( + vm, node, reass, bi0, REASS_FRAGMENT_FORWARD, + reass->ip_proto, reass->l4_src_port, reass->l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); } goto packet_enqueue; } ip4_sv_reass_rc_t rc = - ip4_sv_reass_update (vm, node, rm, rt, reass, bi0); + ip4_sv_reass_update (vm, node, rm, ip0, reass, bi0); switch (rc) { case IP4_SV_REASS_RC_OK: @@ -506,8 +860,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, break; case IP4_SV_REASS_RC_UNSUPP_IP_PROTO: vlib_node_increment_counter (vm, node->node_index, - IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG, - 1); + IP4_ERROR_REASS_UNSUPP_IP_PROT, 1); ip4_sv_reass_free (vm, rm, rt, reass); goto next_packet; break; @@ -519,6 +872,11 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { u32 bi0 = vec_elt (reass->cached_buffers, idx); vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); u32 next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; if (is_feature) { @@ -538,16 +896,23 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; + vnet_buffer (b0)->ip.reass.is_non_first_fragment = + ! !ip4_get_fragment_offset (ip0); vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto; + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + reass->icmp_type_or_tcp_flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + reass->tcp_ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + reass->tcp_seq_number; vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port; vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port; if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - ip4_sv_reass_add_trace (vm, node, rm, reass, bi0, - REASS_FRAGMENT_FORWARD, - reass->ip_proto, - reass->l4_src_port, - reass->l4_dst_port); + ip4_sv_reass_add_trace ( + vm, node, reass, bi0, REASS_FRAGMENT_FORWARD, + reass->ip_proto, reass->l4_src_port, reass->l4_dst_port, + vnet_buffer (b0)->ip.reass.l4_layer_truncated); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, @@ -558,8 +923,6 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, goto next_packet; packet_enqueue: - b0->error = node->errors[error0]; - to_next[0] = bi0; to_next += 1; n_left_to_next -= 1; @@ -568,10 +931,6 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, b0 = vlib_get_buffer (vm, bi0); vnet_feature_next (&next0, b0); } - if (is_custom) - { - next0 = vnet_buffer (b0)->ip.reass.next_index; - } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); @@ -584,6 +943,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_put_next_frame (vm, node, next_index, n_left_to_next); } +done: clib_spinlock_unlock (&rt->lock); return frame->n_vectors; } @@ -599,6 +959,7 @@ VLIB_NODE_FN (ip4_sv_reass_node) (vlib_main_t * vm, vlib_frame_t * frame) { return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ , + false /* is_output_feature */ , false /* is_custom */ ); } @@ -625,6 +986,7 @@ VLIB_NODE_FN (ip4_sv_reass_node_feature) (vlib_main_t * vm, vlib_frame_t * frame) { return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ , + false /* is_output_feature */ , false /* is_custom */ ); } @@ -654,6 +1016,42 @@ VNET_FEATURE_INIT (ip4_sv_reass_feature) = { }; /* *INDENT-ON* */ +VLIB_NODE_FN (ip4_sv_reass_node_output_feature) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return ip4_sv_reass_inline (vm, node, frame, true /* is_feature */ , + true /* is_output_feature */ , + false /* is_custom */ ); +} + + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ip4_sv_reass_node_output_feature) = { + .name = "ip4-sv-reassembly-output-feature", + .vector_size = sizeof (u32), + .format_trace = format_ip4_sv_reass_trace, + .n_errors = ARRAY_LEN (ip4_sv_reass_error_strings), + .error_strings = ip4_sv_reass_error_strings, + .n_next_nodes = IP4_SV_REASSEMBLY_N_NEXT, + .next_nodes = + { + [IP4_SV_REASSEMBLY_NEXT_INPUT] = "ip4-input", + [IP4_SV_REASSEMBLY_NEXT_DROP] = "ip4-drop", + [IP4_SV_REASSEMBLY_NEXT_HANDOFF] = "ip4-sv-reass-feature-hoff", + }, +}; +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VNET_FEATURE_INIT (ip4_sv_reass_output_feature) = { + .arc_name = "ip4-output", + .node_name = "ip4-sv-reassembly-output-feature", + .runs_before = 0, + .runs_after = 0, +}; +/* *INDENT-ON* */ + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip4_sv_reass_custom_node) = { .name = "ip4-sv-reassembly-custom-next", @@ -677,6 +1075,7 @@ VLIB_NODE_FN (ip4_sv_reass_custom_node) (vlib_main_t * vm, vlib_frame_t * frame) { return ip4_sv_reass_inline (vm, node, frame, false /* is_feature */ , + false /* is_output_feature */ , true /* is_custom */ ); } @@ -711,7 +1110,7 @@ typedef struct } ip4_rehash_cb_ctx; #ifndef CLIB_MARCH_VARIANT -static void +static int ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx) { ip4_rehash_cb_ctx *ctx = _ctx; @@ -719,6 +1118,7 @@ ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx) { ctx->failure = 1; } + return (BIHASH_WALK_CONTINUE); } static void @@ -823,6 +1223,7 @@ ip4_sv_reass_init_function (vlib_main_t * vm) vlib_frame_queue_main_init (ip4_sv_reass_node_feature.index, 0); rm->feature_use_refcount_per_intf = NULL; + rm->output_feature_use_refcount_per_intf = NULL; return error; } @@ -831,8 +1232,9 @@ VLIB_INIT_FUNCTION (ip4_sv_reass_init_function); #endif /* CLIB_MARCH_VARIANT */ static uword -ip4_sv_reass_walk_expired (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * f) +ip4_sv_reass_walk_expired (vlib_main_t *vm, + CLIB_UNUSED (vlib_node_runtime_t *node), + CLIB_UNUSED (vlib_frame_t *f)) { ip4_sv_reass_main_t *rm = &ip4_sv_reass_main; uword event_type, *event_data = 0; @@ -847,10 +1249,11 @@ ip4_sv_reass_walk_expired (vlib_main_t * vm, switch (event_type) { - case ~0: /* no events => timeout */ - /* nothing to do here */ - break; + case ~0: + /* no events => timeout */ + /* fallthrough */ case IP4_EVENT_CONFIG_CHANGED: + /* nothing to do here */ break; default: clib_warning ("BUG: event type 0x%wx", event_type); @@ -871,13 +1274,13 @@ ip4_sv_reass_walk_expired (vlib_main_t * vm, vec_reset_length (pool_indexes_to_free); /* *INDENT-OFF* */ - pool_foreach_index (index, rt->pool, ({ + pool_foreach_index (index, rt->pool) { reass = pool_elt_at_index (rt->pool, index); if (now > reass->last_heard + rm->timeout) { vec_add1 (pool_indexes_to_free, index); } - })); + } /* *INDENT-ON* */ int *i; /* *INDENT-OFF* */ @@ -918,9 +1321,8 @@ format_ip4_sv_reass_key (u8 * s, va_list * args) { ip4_sv_reass_key_t *key = va_arg (*args, ip4_sv_reass_key_t *); s = - format (s, - "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u", - key->xx_id, format_ip4_address, &key->src, format_ip4_address, + format (s, "fib_index: %u, src: %U, dst: %U, frag_id: %u, proto: %u", + key->fib_index, format_ip4_address, &key->src, format_ip4_address, &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto); return s; } @@ -980,9 +1382,9 @@ show_ip4_reass (vlib_main_t * vm, if (details) { /* *INDENT-OFF* */ - pool_foreach (reass, rt->pool, { + pool_foreach (reass, rt->pool) { vlib_cli_output (vm, "%U", format_ip4_sv_reass, vm, reass); - }); + } /* *INDENT-ON* */ } sum_reass_n += rt->reass_n; @@ -992,8 +1394,18 @@ show_ip4_reass (vlib_main_t * vm, vlib_cli_output (vm, "Current IP4 reassemblies count: %lu\n", (long unsigned) sum_reass_n); vlib_cli_output (vm, - "Maximum configured concurrent IP4 reassemblies per worker-thread: %lu\n", + "Maximum configured concurrent shallow virtual IP4 reassemblies per worker-thread: %lu\n", (long unsigned) rm->max_reass_n); + vlib_cli_output (vm, + "Maximum configured amount of fragments per shallow " + "virtual IP4 reassembly: %lu\n", + (long unsigned) rm->max_reass_len); + vlib_cli_output (vm, + "Maximum configured shallow virtual IP4 reassembly timeout: %lums\n", + (long unsigned) rm->timeout_ms); + vlib_cli_output (vm, + "Maximum configured shallow virtual IP4 reassembly expire walk interval: %lums\n", + (long unsigned) rm->expire_walk_interval_ms); return 0; } @@ -1009,9 +1421,8 @@ VLIB_CLI_COMMAND (show_ip4_sv_reass_cmd, static) = { vnet_api_error_t ip4_sv_reass_enable_disable (u32 sw_if_index, u8 enable_disable) { - return vnet_feature_enable_disable ("ip4-unicast", - "ip4-sv-reassembly-feature", - sw_if_index, enable_disable, 0, 0); + return ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, + enable_disable); } #endif /* CLIB_MARCH_VARIANT */ @@ -1092,9 +1503,8 @@ ip4_sv_reass_handoff_node_inline (vlib_main_t * vm, ti += 1; b += 1; } - n_enq = - vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices, - frame->n_vectors, 1); + n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from, + thread_indices, frame->n_vectors, 1); if (n_enq < frame->n_vectors) vlib_node_increment_counter (vm, node->node_index, @@ -1176,7 +1586,8 @@ ip4_sv_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable) } else { - --rm->feature_use_refcount_per_intf[sw_if_index]; + if (rm->feature_use_refcount_per_intf[sw_if_index]) + --rm->feature_use_refcount_per_intf[sw_if_index]; if (!rm->feature_use_refcount_per_intf[sw_if_index]) return vnet_feature_enable_disable ("ip4-unicast", "ip4-sv-reassembly-feature", @@ -1191,6 +1602,35 @@ ip4_sv_reass_custom_register_next_node (uword node_index) return vlib_node_add_next (vlib_get_main (), ip4_sv_reass_custom_node.index, node_index); } + +int +ip4_sv_reass_output_enable_disable_with_refcnt (u32 sw_if_index, + int is_enable) +{ + ip4_sv_reass_main_t *rm = &ip4_sv_reass_main; + vec_validate (rm->output_feature_use_refcount_per_intf, sw_if_index); + if (is_enable) + { + if (!rm->output_feature_use_refcount_per_intf[sw_if_index]) + { + ++rm->output_feature_use_refcount_per_intf[sw_if_index]; + return vnet_feature_enable_disable ("ip4-output", + "ip4-sv-reassembly-output-feature", + sw_if_index, 1, 0, 0); + } + ++rm->output_feature_use_refcount_per_intf[sw_if_index]; + } + else + { + if (rm->output_feature_use_refcount_per_intf[sw_if_index]) + --rm->output_feature_use_refcount_per_intf[sw_if_index]; + if (!rm->output_feature_use_refcount_per_intf[sw_if_index]) + return vnet_feature_enable_disable ("ip4-output", + "ip4-sv-reassembly-output-feature", + sw_if_index, 0, 0, 0); + } + return 0; +} #endif /*