X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Fip%2Freass%2Fip4_sv_reass.c;h=e95826380479f940d515b103133730ab976ccb50;hb=refs%2Fchanges%2F31%2F30431%2F2;hp=9b6116c9778055f8f900e58d06efba64321e6e19;hpb=1766ddca4406cbfddb2878990d580a6f9e34a58e;p=vpp.git diff --git a/src/vnet/ip/reass/ip4_sv_reass.c b/src/vnet/ip/reass/ip4_sv_reass.c index 9b6116c9778..e9582638047 100644 --- a/src/vnet/ip/reass/ip4_sv_reass.c +++ b/src/vnet/ip/reass/ip4_sv_reass.c @@ -235,6 +235,13 @@ ip4_sv_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node, u32 ip_proto, u16 l4_src_port, u16 l4_dst_port) { vlib_buffer_t *b = vlib_get_buffer (vm, bi); + if (pool_is_free_index + (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b))) + { + // this buffer's trace is gone + b->flags &= ~VLIB_BUFFER_IS_TRACED; + return; + } ip4_sv_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0])); if (reass) { @@ -305,10 +312,9 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, ip4_sv_reass_kv_t * kv, u8 * do_handoff) { ip4_sv_reass_t *reass = NULL; - f64 now = vlib_time_now (rm->vlib_main); + f64 now = vlib_time_now (vm); - if (!clib_bihash_search_16_8 - (&rm->hash, (clib_bihash_kv_16_8_t *) kv, (clib_bihash_kv_16_8_t *) kv)) + if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv)) { if (vm->thread_index != kv->v.thread_index) { @@ -332,7 +338,7 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, if (rt->reass_n >= rm->max_reass_n && rm->max_reass_n) { - reass = pool_elt_at_index (rt->pool, rt->lru_last); + reass = pool_elt_at_index (rt->pool, rt->lru_first); ip4_sv_reass_free (vm, rm, rt, reass); } @@ -356,13 +362,13 @@ ip4_sv_reass_find_or_create (vlib_main_t * vm, ip4_sv_reass_main_t * rm, rt->lru_first = rt->lru_last = reass - rt->pool; } - reass->key.as_u64[0] = ((clib_bihash_kv_16_8_t *) kv)->key[0]; - reass->key.as_u64[1] = ((clib_bihash_kv_16_8_t *) kv)->key[1]; + reass->key.as_u64[0] = kv->kv.key[0]; + reass->key.as_u64[1] = kv->kv.key[1]; kv->v.reass_index = (reass - rt->pool); kv->v.thread_index = vm->thread_index; reass->last_heard = now; - if (clib_bihash_add_del_16_8 (&rm->hash, (clib_bihash_kv_16_8_t *) kv, 1)) + if (clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 1)) { ip4_sv_reass_free (vm, rm, rt, reass); reass = NULL; @@ -436,6 +442,211 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, n_left_from = frame->n_vectors; next_index = node->cached_next_index; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + vlib_get_buffers (vm, from, bufs, n_left_from); + u16 nexts[VLIB_FRAME_SIZE], *next = nexts; + b = bufs; + + /* optimistic case first - no fragments */ + while (n_left_from >= 2) + { + vlib_buffer_t *b0, *b1; + u32 next0, next1; + b0 = *b; + b++; + b1 = *b; + b++; + + /* Prefetch next iteration. */ + if (PREDICT_TRUE (n_left_from >= 4)) + { + vlib_buffer_t *p2, *p3; + + p2 = *b; + p3 = *(b + 1); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + } + + ip4_header_t *ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); + ip4_header_t *ip1 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b1), + (is_output_feature ? 1 : 0) * + vnet_buffer (b1)-> + ip.save_rewrite_length); + if (PREDICT_FALSE + (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0)) + || (ip4_get_fragment_more (ip1) || ip4_get_fragment_offset (ip1))) + { + // fragment found, go slow path + b -= 2; + if (b - bufs > 0) + { + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + b - bufs); + } + goto slow_path; + } + if (is_feature) + { + vnet_feature_next (&next0, b0); + } + else + { + next0 = is_custom ? vnet_buffer (b0)->ip.reass.next_index : + IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol; + if (IP_PROTOCOL_TCP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip0 + 1))->flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip0 + 1))->ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } + vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1); + vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0); + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 2) - bufs], + REASS_PASSTHROUGH, + vnet_buffer (b0)->ip.reass.ip_proto, + vnet_buffer (b0)->ip.reass.l4_src_port, + vnet_buffer (b0)->ip.reass.l4_dst_port); + } + if (is_feature) + { + vnet_feature_next (&next1, b1); + } + else + { + next1 = is_custom ? vnet_buffer (b1)->ip.reass.next_index : + IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b1)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b1)->ip.reass.ip_proto = ip1->protocol; + if (IP_PROTOCOL_TCP == ip1->protocol) + { + vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip1 + 1))->flags; + vnet_buffer (b1)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip1 + 1))->ack_number; + vnet_buffer (b1)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip1 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip1->protocol) + { + vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip1 + 1))->type; + } + vnet_buffer (b1)->ip.reass.l4_src_port = ip4_get_port (ip1, 1); + vnet_buffer (b1)->ip.reass.l4_dst_port = ip4_get_port (ip1, 0); + if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 1) - bufs], + REASS_PASSTHROUGH, + vnet_buffer (b1)->ip.reass.ip_proto, + vnet_buffer (b1)->ip.reass.l4_src_port, + vnet_buffer (b1)->ip.reass.l4_dst_port); + } + + n_left_from -= 2; + next[0] = next0; + next[1] = next1; + next += 2; + } + + while (n_left_from > 0) + { + vlib_buffer_t *b0; + u32 next0; + b0 = *b; + b++; + + ip4_header_t *ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); + if (PREDICT_FALSE + (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0))) + { + // fragment found, go slow path + b -= 1; + if (b - bufs > 0) + { + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + b - bufs); + } + goto slow_path; + } + if (is_feature) + { + vnet_feature_next (&next0, b0); + } + else + { + next0 = + is_custom ? vnet_buffer (b0)->ip. + reass.next_index : IP4_SV_REASSEMBLY_NEXT_INPUT; + } + vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0; + vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol; + if (IP_PROTOCOL_TCP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((tcp_header_t *) (ip0 + 1))->flags; + vnet_buffer (b0)->ip.reass.tcp_ack_number = + ((tcp_header_t *) (ip0 + 1))->ack_number; + vnet_buffer (b0)->ip.reass.tcp_seq_number = + ((tcp_header_t *) (ip0 + 1))->seq_number; + } + else if (IP_PROTOCOL_ICMP == ip0->protocol) + { + vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = + ((icmp46_header_t *) (ip0 + 1))->type; + } + vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1); + vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0); + if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) + { + ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 1) - bufs], + REASS_PASSTHROUGH, + vnet_buffer (b0)->ip.reass.ip_proto, + vnet_buffer (b0)->ip.reass.l4_src_port, + vnet_buffer (b0)->ip.reass.l4_dst_port); + } + + n_left_from -= 1; + next[0] = next0; + next += 1; + } + + vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts, + frame->n_vectors); + + goto done; + +slow_path: + + from += b - bufs; + while (n_left_from > 0) { vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); @@ -452,7 +663,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, ip4_header_t *ip0 = (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), - is_output_feature * + (is_output_feature ? 1 : 0) * vnet_buffer (b0)-> ip.save_rewrite_length); if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0)) @@ -598,6 +809,11 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { u32 bi0 = vec_elt (reass->cached_buffers, idx); vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + ip0 = + (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0), + (is_output_feature ? 1 : 0) * + vnet_buffer (b0)-> + ip.save_rewrite_length); u32 next0 = IP4_SV_REASSEMBLY_NEXT_INPUT; if (is_feature) { @@ -618,7 +834,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, to_next += 1; n_left_to_next -= 1; vnet_buffer (b0)->ip.reass.is_non_first_fragment = - ! !ip4_get_fragment_offset (vlib_buffer_get_current (b0)); + ! !ip4_get_fragment_offset (ip0); vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto; vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags = reass->icmp_type_or_tcp_flags; @@ -665,6 +881,7 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_put_next_frame (vm, node, next_index, n_left_to_next); } +done: clib_spinlock_unlock (&rt->lock); return frame->n_vectors; } @@ -993,13 +1210,13 @@ ip4_sv_reass_walk_expired (vlib_main_t * vm, vec_reset_length (pool_indexes_to_free); /* *INDENT-OFF* */ - pool_foreach_index (index, rt->pool, ({ + pool_foreach_index (index, rt->pool) { reass = pool_elt_at_index (rt->pool, index); if (now > reass->last_heard + rm->timeout) { vec_add1 (pool_indexes_to_free, index); } - })); + } /* *INDENT-ON* */ int *i; /* *INDENT-OFF* */ @@ -1102,9 +1319,9 @@ show_ip4_reass (vlib_main_t * vm, if (details) { /* *INDENT-OFF* */ - pool_foreach (reass, rt->pool, { + pool_foreach (reass, rt->pool) { vlib_cli_output (vm, "%U", format_ip4_sv_reass, vm, reass); - }); + } /* *INDENT-ON* */ } sum_reass_n += rt->reass_n;