ip4_sv_reass_kv_t * kv, u8 * do_handoff)
{
ip4_sv_reass_t *reass = NULL;
- f64 now = vlib_time_now (rm->vlib_main);
+ f64 now = vlib_time_now (vm);
- if (!clib_bihash_search_16_8
- (&rm->hash, (clib_bihash_kv_16_8_t *) kv, (clib_bihash_kv_16_8_t *) kv))
+ if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv))
{
if (vm->thread_index != kv->v.thread_index)
{
if (rt->reass_n >= rm->max_reass_n && rm->max_reass_n)
{
- reass = pool_elt_at_index (rt->pool, rt->lru_last);
+ reass = pool_elt_at_index (rt->pool, rt->lru_first);
ip4_sv_reass_free (vm, rm, rt, reass);
}
rt->lru_first = rt->lru_last = reass - rt->pool;
}
- reass->key.as_u64[0] = ((clib_bihash_kv_16_8_t *) kv)->key[0];
- reass->key.as_u64[1] = ((clib_bihash_kv_16_8_t *) kv)->key[1];
+ reass->key.as_u64[0] = kv->kv.key[0];
+ reass->key.as_u64[1] = kv->kv.key[1];
kv->v.reass_index = (reass - rt->pool);
kv->v.thread_index = vm->thread_index;
reass->last_heard = now;
- if (clib_bihash_add_del_16_8 (&rm->hash, (clib_bihash_kv_16_8_t *) kv, 1))
+ if (clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 1))
{
ip4_sv_reass_free (vm, rm, rt, reass);
reass = NULL;
n_left_from = frame->n_vectors;
next_index = node->cached_next_index;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+ vlib_get_buffers (vm, from, bufs, n_left_from);
+ u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
+ b = bufs;
+
+ /* optimistic case first - no fragments */
+ while (n_left_from >= 2)
+ {
+ vlib_buffer_t *b0, *b1;
+ u32 next0, next1;
+ b0 = *b;
+ b++;
+ b1 = *b;
+ b++;
+
+ /* Prefetch next iteration. */
+ if (PREDICT_TRUE (n_left_from >= 4))
+ {
+ vlib_buffer_t *p2, *p3;
+
+ p2 = *b;
+ p3 = *(b + 1);
+
+ vlib_prefetch_buffer_header (p2, LOAD);
+ vlib_prefetch_buffer_header (p3, LOAD);
+
+ CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD);
+ }
+
+ ip4_header_t *ip0 =
+ (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
+ (is_output_feature ? 1 : 0) *
+ vnet_buffer (b0)->
+ ip.save_rewrite_length);
+ ip4_header_t *ip1 =
+ (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b1),
+ (is_output_feature ? 1 : 0) *
+ vnet_buffer (b1)->
+ ip.save_rewrite_length);
+ if (PREDICT_FALSE
+ (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0))
+ || (ip4_get_fragment_more (ip1) || ip4_get_fragment_offset (ip1)))
+ {
+ // fragment found, go slow path
+ b -= 2;
+ if (b - bufs > 0)
+ {
+ vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
+ b - bufs);
+ }
+ goto slow_path;
+ }
+ if (is_feature)
+ {
+ vnet_feature_next (&next0, b0);
+ }
+ else
+ {
+ next0 = is_custom ? vnet_buffer (b0)->ip.reass.next_index :
+ IP4_SV_REASSEMBLY_NEXT_INPUT;
+ }
+ vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0;
+ vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol;
+ if (IP_PROTOCOL_TCP == ip0->protocol)
+ {
+ vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
+ ((tcp_header_t *) (ip0 + 1))->flags;
+ vnet_buffer (b0)->ip.reass.tcp_ack_number =
+ ((tcp_header_t *) (ip0 + 1))->ack_number;
+ vnet_buffer (b0)->ip.reass.tcp_seq_number =
+ ((tcp_header_t *) (ip0 + 1))->seq_number;
+ }
+ else if (IP_PROTOCOL_ICMP == ip0->protocol)
+ {
+ vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
+ ((icmp46_header_t *) (ip0 + 1))->type;
+ }
+ vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1);
+ vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0);
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 2) - bufs],
+ REASS_PASSTHROUGH,
+ vnet_buffer (b0)->ip.reass.ip_proto,
+ vnet_buffer (b0)->ip.reass.l4_src_port,
+ vnet_buffer (b0)->ip.reass.l4_dst_port);
+ }
+ if (is_feature)
+ {
+ vnet_feature_next (&next1, b1);
+ }
+ else
+ {
+ next1 = is_custom ? vnet_buffer (b1)->ip.reass.next_index :
+ IP4_SV_REASSEMBLY_NEXT_INPUT;
+ }
+ vnet_buffer (b1)->ip.reass.is_non_first_fragment = 0;
+ vnet_buffer (b1)->ip.reass.ip_proto = ip1->protocol;
+ if (IP_PROTOCOL_TCP == ip1->protocol)
+ {
+ vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags =
+ ((tcp_header_t *) (ip1 + 1))->flags;
+ vnet_buffer (b1)->ip.reass.tcp_ack_number =
+ ((tcp_header_t *) (ip1 + 1))->ack_number;
+ vnet_buffer (b1)->ip.reass.tcp_seq_number =
+ ((tcp_header_t *) (ip1 + 1))->seq_number;
+ }
+ else if (IP_PROTOCOL_ICMP == ip1->protocol)
+ {
+ vnet_buffer (b1)->ip.reass.icmp_type_or_tcp_flags =
+ ((icmp46_header_t *) (ip1 + 1))->type;
+ }
+ vnet_buffer (b1)->ip.reass.l4_src_port = ip4_get_port (ip1, 1);
+ vnet_buffer (b1)->ip.reass.l4_dst_port = ip4_get_port (ip1, 0);
+ if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 1) - bufs],
+ REASS_PASSTHROUGH,
+ vnet_buffer (b1)->ip.reass.ip_proto,
+ vnet_buffer (b1)->ip.reass.l4_src_port,
+ vnet_buffer (b1)->ip.reass.l4_dst_port);
+ }
+
+ n_left_from -= 2;
+ next[0] = next0;
+ next[1] = next1;
+ next += 2;
+ }
+
+ while (n_left_from > 0)
+ {
+ vlib_buffer_t *b0;
+ u32 next0;
+ b0 = *b;
+ b++;
+
+ ip4_header_t *ip0 =
+ (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
+ (is_output_feature ? 1 : 0) *
+ vnet_buffer (b0)->
+ ip.save_rewrite_length);
+ if (PREDICT_FALSE
+ (ip4_get_fragment_more (ip0) || ip4_get_fragment_offset (ip0)))
+ {
+ // fragment found, go slow path
+ b -= 1;
+ if (b - bufs > 0)
+ {
+ vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
+ b - bufs);
+ }
+ goto slow_path;
+ }
+ if (is_feature)
+ {
+ vnet_feature_next (&next0, b0);
+ }
+ else
+ {
+ next0 =
+ is_custom ? vnet_buffer (b0)->ip.
+ reass.next_index : IP4_SV_REASSEMBLY_NEXT_INPUT;
+ }
+ vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0;
+ vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol;
+ if (IP_PROTOCOL_TCP == ip0->protocol)
+ {
+ vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
+ ((tcp_header_t *) (ip0 + 1))->flags;
+ vnet_buffer (b0)->ip.reass.tcp_ack_number =
+ ((tcp_header_t *) (ip0 + 1))->ack_number;
+ vnet_buffer (b0)->ip.reass.tcp_seq_number =
+ ((tcp_header_t *) (ip0 + 1))->seq_number;
+ }
+ else if (IP_PROTOCOL_ICMP == ip0->protocol)
+ {
+ vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
+ ((icmp46_header_t *) (ip0 + 1))->type;
+ }
+ vnet_buffer (b0)->ip.reass.l4_src_port = ip4_get_port (ip0, 1);
+ vnet_buffer (b0)->ip.reass.l4_dst_port = ip4_get_port (ip0, 0);
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ ip4_sv_reass_add_trace (vm, node, rm, NULL, from[(b - 1) - bufs],
+ REASS_PASSTHROUGH,
+ vnet_buffer (b0)->ip.reass.ip_proto,
+ vnet_buffer (b0)->ip.reass.l4_src_port,
+ vnet_buffer (b0)->ip.reass.l4_dst_port);
+ }
+
+ n_left_from -= 1;
+ next[0] = next0;
+ next += 1;
+ }
+
+ vlib_buffer_enqueue_to_next (vm, node, from, (u16 *) nexts,
+ frame->n_vectors);
+
+ goto done;
+
+slow_path:
+
+ from += b - bufs;
+
while (n_left_from > 0)
{
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
ip4_header_t *ip0 =
(ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
- is_output_feature *
+ (is_output_feature ? 1 : 0) *
vnet_buffer (b0)->
ip.save_rewrite_length);
if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
{
next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
}
- vnet_buffer (b0)->ip.reass.save_rewrite_length =
- vnet_buffer (b0)->ip.save_rewrite_length;
vnet_buffer (b0)->ip.reass.is_non_first_fragment = 0;
vnet_buffer (b0)->ip.reass.ip_proto = ip0->protocol;
if (IP_PROTOCOL_TCP == ip0->protocol)
{
next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
error0 = IP4_ERROR_REASS_MALFORMED_PACKET;
+ b0->error = node->errors[error0];
goto packet_enqueue;
}
ip4_sv_reass_kv_t kv;
{
next0 = IP4_SV_REASSEMBLY_NEXT_DROP;
error0 = IP4_ERROR_REASS_LIMIT_REACHED;
+ b0->error = node->errors[error0];
goto packet_enqueue;
}
{
next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
}
- vnet_buffer (b0)->ip.reass.save_rewrite_length =
- vnet_buffer (b0)->ip.save_rewrite_length;
vnet_buffer (b0)->ip.reass.is_non_first_fragment =
! !fragment_first;
vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
reass->tcp_seq_number;
vnet_buffer (b0)->ip.reass.l4_src_port = reass->l4_src_port;
vnet_buffer (b0)->ip.reass.l4_dst_port = reass->l4_dst_port;
- error0 = IP4_ERROR_NONE;
if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
{
ip4_sv_reass_add_trace (vm, node, rm, reass, bi0,
{
u32 bi0 = vec_elt (reass->cached_buffers, idx);
vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
+ ip0 =
+ (ip4_header_t *) u8_ptr_add (vlib_buffer_get_current (b0),
+ (is_output_feature ? 1 : 0) *
+ vnet_buffer (b0)->
+ ip.save_rewrite_length);
u32 next0 = IP4_SV_REASSEMBLY_NEXT_INPUT;
if (is_feature)
{
to_next[0] = bi0;
to_next += 1;
n_left_to_next -= 1;
- ASSERT (vnet_buffer (b0)->ip.save_rewrite_length < (2 << 14));
- vnet_buffer (b0)->ip.reass.save_rewrite_length =
- vnet_buffer (b0)->ip.save_rewrite_length;
vnet_buffer (b0)->ip.reass.is_non_first_fragment =
- ! !ip4_get_fragment_offset (vlib_buffer_get_current (b0));
+ ! !ip4_get_fragment_offset (ip0);
vnet_buffer (b0)->ip.reass.ip_proto = reass->ip_proto;
vnet_buffer (b0)->ip.reass.icmp_type_or_tcp_flags =
reass->icmp_type_or_tcp_flags;
goto next_packet;
packet_enqueue:
- b0->error = node->errors[error0];
-
to_next[0] = bi0;
to_next += 1;
n_left_to_next -= 1;
vlib_put_next_frame (vm, node, next_index, n_left_to_next);
}
+done:
clib_spinlock_unlock (&rt->lock);
return frame->n_vectors;
}
vec_reset_length (pool_indexes_to_free);
/* *INDENT-OFF* */
- pool_foreach_index (index, rt->pool, ({
+ pool_foreach_index (index, rt->pool) {
reass = pool_elt_at_index (rt->pool, index);
if (now > reass->last_heard + rm->timeout)
{
vec_add1 (pool_indexes_to_free, index);
}
- }));
+ }
/* *INDENT-ON* */
int *i;
/* *INDENT-OFF* */
if (details)
{
/* *INDENT-OFF* */
- pool_foreach (reass, rt->pool, {
+ pool_foreach (reass, rt->pool) {
vlib_cli_output (vm, "%U", format_ip4_sv_reass, vm, reass);
- });
+ }
/* *INDENT-ON* */
}
sum_reass_n += rt->reass_n;
vlib_cli_output (vm, "Current IP4 reassemblies count: %lu\n",
(long unsigned) sum_reass_n);
vlib_cli_output (vm,
- "Maximum configured concurrent IP4 reassemblies per worker-thread: %lu\n",
+ "Maximum configured concurrent shallow virtual IP4 reassemblies per worker-thread: %lu\n",
(long unsigned) rm->max_reass_n);
+ vlib_cli_output (vm,
+ "Maximum configured shallow virtual IP4 reassembly timeout: %lums\n",
+ (long unsigned) rm->timeout_ms);
+ vlib_cli_output (vm,
+ "Maximum configured shallow virtual IP4 reassembly expire walk interval: %lums\n",
+ (long unsigned) rm->expire_walk_interval_ms);
return 0;
}
Code Review / vpp.git / blobdiff