2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
18 * @brief IPv4 Full Reassembly.
20 * This file contains the source code for IPv4 full reassembly.
23 #include <vppinfra/vec.h>
24 #include <vnet/vnet.h>
25 #include <vnet/ip/ip.h>
26 #include <vppinfra/fifo.h>
27 #include <vppinfra/bihash_16_8.h>
28 #include <vnet/ip/reass/ip4_full_reass.h>
31 #define MSEC_PER_SEC 1000
32 #define IP4_REASS_TIMEOUT_DEFAULT_MS 200
34 /* As there are only 1024 reass context per thread, either the DDOS attacks
35 * or fractions of real timeouts, would consume these contexts quickly and
36 * running out context space and unable to perform reassembly */
37 #define IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 50 // 50 ms default
38 #define IP4_REASS_MAX_REASSEMBLIES_DEFAULT 1024
39 #define IP4_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
40 #define IP4_REASS_HT_LOAD_FACTOR (0.75)
42 #define IP4_REASS_DEBUG_BUFFERS 0
43 #if IP4_REASS_DEBUG_BUFFERS
44 #define IP4_REASS_DEBUG_BUFFER(bi, what) \
48 printf (#what "buffer %u", _bi); \
49 vlib_buffer_t *_b = vlib_get_buffer (vm, _bi); \
50 while (_b->flags & VLIB_BUFFER_NEXT_PRESENT) \
52 _bi = _b->next_buffer; \
53 printf ("[%u]", _bi); \
54 _b = vlib_get_buffer (vm, _bi); \
61 #define IP4_REASS_DEBUG_BUFFER(...)
67 IP4_REASS_RC_TOO_MANY_FRAGMENTS,
68 IP4_REASS_RC_INTERNAL_ERROR,
71 } ip4_full_reass_rc_t;
88 } ip4_full_reass_key_t;
95 u32 memory_owner_thread_index;
98 } ip4_full_reass_val_t;
104 ip4_full_reass_key_t k;
105 ip4_full_reass_val_t v;
107 clib_bihash_kv_16_8_t kv;
108 } ip4_full_reass_kv_t;
111 ip4_full_reass_buffer_get_data_offset (vlib_buffer_t * b)
113 vnet_buffer_opaque_t *vnb = vnet_buffer (b);
114 return vnb->ip.reass.range_first - vnb->ip.reass.fragment_first;
118 ip4_full_reass_buffer_get_data_len (vlib_buffer_t * b)
120 vnet_buffer_opaque_t *vnb = vnet_buffer (b);
121 return clib_min (vnb->ip.reass.range_last, vnb->ip.reass.fragment_last) -
122 (vnb->ip.reass.fragment_first +
123 ip4_full_reass_buffer_get_data_offset (b)) + 1;
129 ip4_full_reass_key_t key;
130 // time when last packet was received
132 // internal id of this reassembly
134 // buffer index of first buffer in this reassembly context
136 // last octet of packet, ~0 until fragment without more_fragments arrives
137 u32 last_packet_octet;
138 // length of data collected so far
140 // trace operation counter
141 u32 trace_op_counter;
142 // next index - used by non-feature node
144 // error next index - used by custom apps (~0 if not used)
145 u32 error_next_index;
146 // minimum fragment length for this reassembly - used to estimate MTU
147 u16 min_fragment_length;
148 // number of fragments in this reassembly
150 // thread owning memory for this context (whose pool contains this ctx)
151 u32 memory_owner_thread_index;
152 // thread which received fragment with offset 0 and which sends out the
153 // completed reassembly
154 u32 sendout_thread_index;
159 ip4_full_reass_t *pool;
162 // for pacing the main thread timeouts
164 clib_spinlock_t lock;
165 } ip4_full_reass_per_thread_t;
172 u32 expire_walk_interval_ms;
173 // maximum number of fragments in one reassembly
175 // maximum number of reassemblies
179 clib_bihash_16_8_t hash;
181 ip4_full_reass_per_thread_t *per_thread_data;
184 vlib_main_t *vlib_main;
186 u32 ip4_full_reass_expire_node_idx;
188 /** Worker handoff */
191 u32 fq_feature_index;
194 // reference count for enabling/disabling feature - per interface
195 u32 *feature_use_refcount_per_intf;
197 // whether local fragmented packets are reassembled or not
198 int is_local_reass_enabled;
199 } ip4_full_reass_main_t;
201 extern ip4_full_reass_main_t ip4_full_reass_main;
203 #ifndef CLIB_MARCH_VARIANT
204 ip4_full_reass_main_t ip4_full_reass_main;
205 #endif /* CLIB_MARCH_VARIANT */
209 IP4_FULL_REASS_NEXT_INPUT,
210 IP4_FULL_REASS_NEXT_DROP,
211 IP4_FULL_REASS_NEXT_HANDOFF,
212 IP4_FULL_REASS_N_NEXT,
213 } ip4_full_reass_next_t;
220 } ip4_full_reass_node_type_t;
231 } ip4_full_reass_trace_operation_e;
241 } ip4_full_reass_range_trace_t;
245 ip4_full_reass_trace_operation_e action;
247 ip4_full_reass_range_trace_t trace_range;
255 bool is_after_handoff;
256 ip4_header_t ip4_header;
257 } ip4_full_reass_trace_t;
259 extern vlib_node_registration_t ip4_full_reass_node;
260 extern vlib_node_registration_t ip4_full_reass_node_feature;
261 extern vlib_node_registration_t ip4_full_reass_node_custom;
264 ip4_full_reass_trace_details (vlib_main_t * vm, u32 bi,
265 ip4_full_reass_range_trace_t * trace)
267 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
268 vnet_buffer_opaque_t *vnb = vnet_buffer (b);
269 trace->range_first = vnb->ip.reass.range_first;
270 trace->range_last = vnb->ip.reass.range_last;
271 trace->data_offset = ip4_full_reass_buffer_get_data_offset (b);
272 trace->data_len = ip4_full_reass_buffer_get_data_len (b);
273 trace->range_bi = bi;
277 format_ip4_full_reass_range_trace (u8 * s, va_list * args)
279 ip4_full_reass_range_trace_t *trace =
280 va_arg (*args, ip4_full_reass_range_trace_t *);
282 format (s, "range: [%u, %u], off %d, len %u, bi %u", trace->range_first,
283 trace->range_last, trace->data_offset, trace->data_len,
289 format_ip4_full_reass_trace (u8 * s, va_list * args)
291 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
292 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
293 ip4_full_reass_trace_t *t = va_arg (*args, ip4_full_reass_trace_t *);
295 if (~0 != t->reass_id)
297 if (t->is_after_handoff)
300 format (s, "%U\n", format_ip4_header, &t->ip4_header,
301 sizeof (t->ip4_header));
305 format (s, "%Ureass id: %u, op id: %u, ", format_white_space, indent,
306 t->reass_id, t->op_id);
307 indent = format_get_indent (s);
310 "first bi: %u, data len: %u, ip/fragment[%u, %u]",
311 t->trace_range.first_bi, t->total_data_len, t->fragment_first,
317 s = format (s, "\n%Ushrink %U by %u", format_white_space, indent,
318 format_ip4_full_reass_range_trace, &t->trace_range,
322 s = format (s, "\n%Udiscard %U", format_white_space, indent,
323 format_ip4_full_reass_range_trace, &t->trace_range);
326 s = format (s, "\n%Unew %U", format_white_space, indent,
327 format_ip4_full_reass_range_trace, &t->trace_range);
330 s = format (s, "\n%Uoverlapping/ignored %U", format_white_space, indent,
331 format_ip4_full_reass_range_trace, &t->trace_range);
334 s = format (s, "\n%Ufinalize reassembly", format_white_space, indent);
338 format (s, "handoff from thread #%u to thread #%u", t->thread_id,
342 s = format (s, "passthrough - not a fragment");
349 ip4_full_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
350 ip4_full_reass_t * reass, u32 bi,
351 ip4_full_reass_trace_operation_e action,
352 u32 size_diff, u32 thread_id_to)
354 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
355 vnet_buffer_opaque_t *vnb = vnet_buffer (b);
356 if (pool_is_free_index
357 (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b)))
359 // this buffer's trace is gone
360 b->flags &= ~VLIB_BUFFER_IS_TRACED;
363 bool is_after_handoff = false;
364 if (vlib_buffer_get_trace_thread (b) != vm->thread_index)
366 is_after_handoff = true;
368 ip4_full_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
369 t->is_after_handoff = is_after_handoff;
370 if (t->is_after_handoff)
372 clib_memcpy (&t->ip4_header, vlib_buffer_get_current (b),
373 clib_min (sizeof (t->ip4_header), b->current_length));
377 t->reass_id = reass->id;
378 t->op_id = reass->trace_op_counter;
379 t->trace_range.first_bi = reass->first_bi;
380 t->total_data_len = reass->data_len;
381 ++reass->trace_op_counter;
387 t->trace_range.first_bi = 0;
388 t->total_data_len = 0;
391 ip4_full_reass_trace_details (vm, bi, &t->trace_range);
392 t->size_diff = size_diff;
393 t->thread_id = vm->thread_index;
394 t->thread_id_to = thread_id_to;
395 t->fragment_first = vnb->ip.reass.fragment_first;
396 t->fragment_last = vnb->ip.reass.fragment_last;
399 s = format (s, "%U", format_ip4_full_reass_trace, NULL, NULL, t);
400 printf ("%.*s\n", vec_len (s), s);
402 vec_reset_length (s);
407 ip4_full_reass_free_ctx (ip4_full_reass_per_thread_t * rt,
408 ip4_full_reass_t * reass)
410 pool_put (rt->pool, reass);
415 ip4_full_reass_free (ip4_full_reass_main_t * rm,
416 ip4_full_reass_per_thread_t * rt,
417 ip4_full_reass_t * reass)
419 clib_bihash_kv_16_8_t kv;
420 kv.key[0] = reass->key.as_u64[0];
421 kv.key[1] = reass->key.as_u64[1];
422 clib_bihash_add_del_16_8 (&rm->hash, &kv, 0);
423 return ip4_full_reass_free_ctx (rt, reass);
426 /* n_left_to_next, and to_next are taken as input params, as this function
427 * could be called from a graphnode, where its managing local copy of these
428 * variables, and ignoring those and still trying to enqueue the buffers
429 * with local variables would cause either buffer leak or corruption */
431 ip4_full_reass_drop_all (vlib_main_t *vm, vlib_node_runtime_t *node,
432 ip4_full_reass_t *reass, u32 *n_left_to_next,
435 u32 range_bi = reass->first_bi;
436 vlib_buffer_t *range_b;
437 vnet_buffer_opaque_t *range_vnb;
440 while (~0 != range_bi)
442 range_b = vlib_get_buffer (vm, range_bi);
443 range_vnb = vnet_buffer (range_b);
447 vec_add1 (to_free, range_bi);
450 range_bi = range_vnb->ip.reass.next_range_bi;
453 /* send to next_error_index */
454 if (~0 != reass->error_next_index &&
455 reass->error_next_index < node->n_next_nodes)
459 next_index = reass->error_next_index;
462 /* record number of packets sent to custom app */
463 vlib_node_increment_counter (vm, node->node_index,
464 IP4_ERROR_REASS_TO_CUSTOM_APP,
467 while (vec_len (to_free) > 0)
469 vlib_get_next_frame (vm, node, next_index, *to_next,
472 while (vec_len (to_free) > 0 && (*n_left_to_next) > 0)
474 bi = vec_pop (to_free);
478 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
479 if ((b->flags & VLIB_BUFFER_IS_TRACED))
481 ip4_full_reass_add_trace (vm, node, reass, bi,
482 RANGE_DISCARD, 0, ~0);
486 (*n_left_to_next) -= 1;
489 vlib_put_next_frame (vm, node, next_index, (*n_left_to_next));
494 vlib_buffer_free (vm, to_free, vec_len (to_free));
500 sanitize_reass_buffers_add_missing (vlib_main_t *vm, ip4_full_reass_t *reass,
503 u32 range_bi = reass->first_bi;
504 vlib_buffer_t *range_b;
505 vnet_buffer_opaque_t *range_vnb;
507 while (~0 != range_bi)
509 range_b = vlib_get_buffer (vm, range_bi);
510 range_vnb = vnet_buffer (range_b);
516 if (range_b->flags & VLIB_BUFFER_NEXT_PRESENT)
519 vlib_buffer_t *_b = vlib_get_buffer (vm, _bi);
520 while (_b->flags & VLIB_BUFFER_NEXT_PRESENT)
522 if (_b->next_buffer != range_vnb->ip.reass.next_range_bi)
524 _bi = _b->next_buffer;
525 _b = vlib_get_buffer (vm, _bi);
529 _b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
534 range_bi = range_vnb->ip.reass.next_range_bi;
539 vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
540 vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
541 if (~0 != reass->first_bi)
543 fvnb->ip.reass.next_range_bi = reass->first_bi;
544 reass->first_bi = *bi0;
548 reass->first_bi = *bi0;
549 fvnb->ip.reass.next_range_bi = ~0;
556 ip4_full_reass_init (ip4_full_reass_t * reass)
558 reass->first_bi = ~0;
559 reass->last_packet_octet = ~0;
561 reass->next_index = ~0;
562 reass->error_next_index = ~0;
565 always_inline ip4_full_reass_t *
566 ip4_full_reass_find_or_create (vlib_main_t *vm, vlib_node_runtime_t *node,
567 ip4_full_reass_main_t *rm,
568 ip4_full_reass_per_thread_t *rt,
569 ip4_full_reass_kv_t *kv, u8 *do_handoff,
570 u32 *n_left_to_next, u32 **to_next)
572 ip4_full_reass_t *reass;
578 now = vlib_time_now (vm);
579 if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv))
581 if (vm->thread_index != kv->v.memory_owner_thread_index)
587 pool_elt_at_index (rm->per_thread_data
588 [kv->v.memory_owner_thread_index].pool,
591 if (now > reass->last_heard + rm->timeout)
593 vlib_node_increment_counter (vm, node->node_index,
594 IP4_ERROR_REASS_TIMEOUT, 1);
595 ip4_full_reass_drop_all (vm, node, reass, n_left_to_next, to_next);
596 ip4_full_reass_free (rm, rt, reass);
603 reass->last_heard = now;
607 if (rt->reass_n >= rm->max_reass_n)
614 pool_get (rt->pool, reass);
615 clib_memset (reass, 0, sizeof (*reass));
616 reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
617 reass->memory_owner_thread_index = vm->thread_index;
619 ip4_full_reass_init (reass);
623 reass->key.as_u64[0] = kv->kv.key[0];
624 reass->key.as_u64[1] = kv->kv.key[1];
625 kv->v.reass_index = (reass - rt->pool);
626 kv->v.memory_owner_thread_index = vm->thread_index;
627 reass->last_heard = now;
629 int rv = clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 2);
632 ip4_full_reass_free_ctx (rt, reass);
634 // if other worker created a context already work with the other copy
642 always_inline ip4_full_reass_rc_t
643 ip4_full_reass_finalize (vlib_main_t * vm, vlib_node_runtime_t * node,
644 ip4_full_reass_main_t * rm,
645 ip4_full_reass_per_thread_t * rt,
646 ip4_full_reass_t * reass, u32 * bi0,
647 u32 * next0, u32 * error0, bool is_custom)
649 vlib_buffer_t *first_b = vlib_get_buffer (vm, reass->first_bi);
650 vlib_buffer_t *last_b = NULL;
651 u32 sub_chain_bi = reass->first_bi;
652 u32 total_length = 0;
656 u32 tmp_bi = sub_chain_bi;
657 vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
658 ip4_header_t *ip = vlib_buffer_get_current (tmp);
659 vnet_buffer_opaque_t *vnb = vnet_buffer (tmp);
660 if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
661 !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
663 return IP4_REASS_RC_INTERNAL_ERROR;
666 u32 data_len = ip4_full_reass_buffer_get_data_len (tmp);
668 ip4_header_bytes (ip) + ip4_full_reass_buffer_get_data_offset (tmp);
670 vlib_buffer_length_in_chain (vm, tmp) - trim_front - data_len;
671 if (tmp_bi == reass->first_bi)
673 /* first buffer - keep ip4 header */
674 if (0 != ip4_full_reass_buffer_get_data_offset (tmp))
676 return IP4_REASS_RC_INTERNAL_ERROR;
679 trim_end = vlib_buffer_length_in_chain (vm, tmp) - data_len -
680 ip4_header_bytes (ip);
681 if (!(vlib_buffer_length_in_chain (vm, tmp) - trim_end > 0))
683 return IP4_REASS_RC_INTERNAL_ERROR;
687 vlib_buffer_length_in_chain (vm, tmp) - trim_front - trim_end;
693 if (trim_front > tmp->current_length)
695 /* drop whole buffer */
696 u32 to_be_freed_bi = tmp_bi;
697 trim_front -= tmp->current_length;
698 if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
700 return IP4_REASS_RC_INTERNAL_ERROR;
702 tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
703 tmp_bi = tmp->next_buffer;
704 tmp->next_buffer = 0;
705 tmp = vlib_get_buffer (vm, tmp_bi);
706 vlib_buffer_free_one (vm, to_be_freed_bi);
711 vlib_buffer_advance (tmp, trim_front);
719 last_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
720 last_b->next_buffer = tmp_bi;
723 if (keep_data <= tmp->current_length)
725 tmp->current_length = keep_data;
730 keep_data -= tmp->current_length;
731 if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
733 return IP4_REASS_RC_INTERNAL_ERROR;
736 total_length += tmp->current_length;
737 if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
739 tmp_bi = tmp->next_buffer;
740 tmp = vlib_get_buffer (vm, tmp->next_buffer);
749 u32 to_be_freed_bi = tmp_bi;
750 if (reass->first_bi == tmp_bi)
752 return IP4_REASS_RC_INTERNAL_ERROR;
754 if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
756 tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
757 tmp_bi = tmp->next_buffer;
758 tmp->next_buffer = 0;
759 tmp = vlib_get_buffer (vm, tmp_bi);
760 vlib_buffer_free_one (vm, to_be_freed_bi);
764 tmp->next_buffer = 0;
765 vlib_buffer_free_one (vm, to_be_freed_bi);
771 vnet_buffer (vlib_get_buffer (vm, sub_chain_bi))->ip.
774 while (~0 != sub_chain_bi);
778 return IP4_REASS_RC_INTERNAL_ERROR;
780 last_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
782 if (total_length < first_b->current_length)
784 return IP4_REASS_RC_INTERNAL_ERROR;
786 total_length -= first_b->current_length;
787 first_b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
788 first_b->total_length_not_including_first_buffer = total_length;
789 ip4_header_t *ip = vlib_buffer_get_current (first_b);
790 ip->flags_and_fragment_offset = 0;
791 ip->length = clib_host_to_net_u16 (first_b->current_length + total_length);
792 ip->checksum = ip4_header_checksum (ip);
793 if (!vlib_buffer_chain_linearize (vm, first_b))
795 return IP4_REASS_RC_NO_BUF;
797 // reset to reconstruct the mbuf linking
798 first_b->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
799 if (PREDICT_FALSE (first_b->flags & VLIB_BUFFER_IS_TRACED))
801 ip4_full_reass_add_trace (vm, node, reass, reass->first_bi, FINALIZE, 0,
804 // following code does a hexdump of packet fragments to stdout ...
807 u32 bi = reass->first_bi;
811 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
812 s = format (s, "%u: %U\n", bi, format_hexdump,
813 vlib_buffer_get_current (b), b->current_length);
814 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
823 printf ("%.*s\n", vec_len (s), s);
830 *bi0 = reass->first_bi;
833 *next0 = IP4_FULL_REASS_NEXT_INPUT;
837 *next0 = reass->next_index;
839 vnet_buffer (first_b)->ip.reass.estimated_mtu = reass->min_fragment_length;
841 /* Keep track of number of successfully reassembled packets and number of
842 * fragments reassembled */
843 vlib_node_increment_counter (vm, node->node_index, IP4_ERROR_REASS_SUCCESS,
846 vlib_node_increment_counter (vm, node->node_index,
847 IP4_ERROR_REASS_FRAGMENTS_REASSEMBLED,
850 *error0 = IP4_ERROR_NONE;
851 ip4_full_reass_free (rm, rt, reass);
853 return IP4_REASS_RC_OK;
856 always_inline ip4_full_reass_rc_t
857 ip4_full_reass_insert_range_in_chain (vlib_main_t * vm,
858 ip4_full_reass_t * reass,
859 u32 prev_range_bi, u32 new_next_bi)
861 vlib_buffer_t *new_next_b = vlib_get_buffer (vm, new_next_bi);
862 vnet_buffer_opaque_t *new_next_vnb = vnet_buffer (new_next_b);
863 if (~0 != prev_range_bi)
865 vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
866 vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
867 new_next_vnb->ip.reass.next_range_bi = prev_vnb->ip.reass.next_range_bi;
868 prev_vnb->ip.reass.next_range_bi = new_next_bi;
872 if (~0 != reass->first_bi)
874 new_next_vnb->ip.reass.next_range_bi = reass->first_bi;
876 reass->first_bi = new_next_bi;
878 vnet_buffer_opaque_t *vnb = vnet_buffer (new_next_b);
879 if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
880 !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
882 return IP4_REASS_RC_INTERNAL_ERROR;
884 reass->data_len += ip4_full_reass_buffer_get_data_len (new_next_b);
885 return IP4_REASS_RC_OK;
888 always_inline ip4_full_reass_rc_t
889 ip4_full_reass_remove_range_from_chain (vlib_main_t * vm,
890 vlib_node_runtime_t * node,
891 ip4_full_reass_t * reass,
892 u32 prev_range_bi, u32 discard_bi)
894 vlib_buffer_t *discard_b = vlib_get_buffer (vm, discard_bi);
895 vnet_buffer_opaque_t *discard_vnb = vnet_buffer (discard_b);
896 if (~0 != prev_range_bi)
898 vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
899 vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
900 if (!(prev_vnb->ip.reass.next_range_bi == discard_bi))
902 return IP4_REASS_RC_INTERNAL_ERROR;
904 prev_vnb->ip.reass.next_range_bi = discard_vnb->ip.reass.next_range_bi;
908 reass->first_bi = discard_vnb->ip.reass.next_range_bi;
910 vnet_buffer_opaque_t *vnb = vnet_buffer (discard_b);
911 if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
912 !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
914 return IP4_REASS_RC_INTERNAL_ERROR;
916 reass->data_len -= ip4_full_reass_buffer_get_data_len (discard_b);
919 u32 to_be_freed_bi = discard_bi;
920 if (PREDICT_FALSE (discard_b->flags & VLIB_BUFFER_IS_TRACED))
922 ip4_full_reass_add_trace (vm, node, reass, discard_bi, RANGE_DISCARD,
925 if (discard_b->flags & VLIB_BUFFER_NEXT_PRESENT)
927 discard_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
928 discard_bi = discard_b->next_buffer;
929 discard_b->next_buffer = 0;
930 discard_b = vlib_get_buffer (vm, discard_bi);
931 vlib_buffer_free_one (vm, to_be_freed_bi);
935 discard_b->next_buffer = 0;
936 vlib_buffer_free_one (vm, to_be_freed_bi);
940 return IP4_REASS_RC_OK;
943 always_inline ip4_full_reass_rc_t
944 ip4_full_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node,
945 ip4_full_reass_main_t * rm,
946 ip4_full_reass_per_thread_t * rt,
947 ip4_full_reass_t * reass, u32 * bi0, u32 * next0,
948 u32 * error0, bool is_custom, u32 * handoff_thread_idx)
950 vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
951 vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
954 // store (error_)next_index before it's overwritten
955 reass->next_index = fvnb->ip.reass.next_index;
956 reass->error_next_index = fvnb->ip.reass.error_next_index;
958 ip4_full_reass_rc_t rc = IP4_REASS_RC_OK;
960 ip4_header_t *fip = vlib_buffer_get_current (fb);
961 const u32 fragment_first = ip4_get_fragment_offset_bytes (fip);
962 const u32 fragment_length =
963 clib_net_to_host_u16 (fip->length) - ip4_header_bytes (fip);
964 const u32 fragment_last = fragment_first + fragment_length - 1;
965 fvnb->ip.reass.fragment_first = fragment_first;
966 fvnb->ip.reass.fragment_last = fragment_last;
967 int more_fragments = ip4_get_fragment_more (fip);
968 u32 candidate_range_bi = reass->first_bi;
969 u32 prev_range_bi = ~0;
970 fvnb->ip.reass.range_first = fragment_first;
971 fvnb->ip.reass.range_last = fragment_last;
972 fvnb->ip.reass.next_range_bi = ~0;
975 reass->last_packet_octet = fragment_last;
977 if (~0 == reass->first_bi)
979 // starting a new reassembly
981 ip4_full_reass_insert_range_in_chain (vm, reass, prev_range_bi, *bi0);
982 if (IP4_REASS_RC_OK != rc)
986 if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
988 ip4_full_reass_add_trace (vm, node, reass, *bi0, RANGE_NEW, 0, ~0);
991 reass->min_fragment_length = clib_net_to_host_u16 (fip->length);
992 reass->fragments_n = 1;
993 return IP4_REASS_RC_OK;
995 reass->min_fragment_length =
996 clib_min (clib_net_to_host_u16 (fip->length),
997 fvnb->ip.reass.estimated_mtu);
998 while (~0 != candidate_range_bi)
1000 vlib_buffer_t *candidate_b = vlib_get_buffer (vm, candidate_range_bi);
1001 vnet_buffer_opaque_t *candidate_vnb = vnet_buffer (candidate_b);
1002 if (fragment_first > candidate_vnb->ip.reass.range_last)
1004 // this fragments starts after candidate range
1005 prev_range_bi = candidate_range_bi;
1006 candidate_range_bi = candidate_vnb->ip.reass.next_range_bi;
1007 if (candidate_vnb->ip.reass.range_last < fragment_last &&
1008 ~0 == candidate_range_bi)
1010 // special case - this fragment falls beyond all known ranges
1011 rc = ip4_full_reass_insert_range_in_chain (vm, reass,
1012 prev_range_bi, *bi0);
1013 if (IP4_REASS_RC_OK != rc)
1022 if (fragment_last < candidate_vnb->ip.reass.range_first)
1024 // this fragment ends before candidate range without any overlap
1025 rc = ip4_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
1027 if (IP4_REASS_RC_OK != rc)
1035 if (fragment_first >= candidate_vnb->ip.reass.range_first &&
1036 fragment_last <= candidate_vnb->ip.reass.range_last)
1038 // this fragment is a (sub)part of existing range, ignore it
1039 if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
1041 ip4_full_reass_add_trace (vm, node, reass, *bi0,
1042 RANGE_OVERLAP, 0, ~0);
1046 int discard_candidate = 0;
1047 if (fragment_first < candidate_vnb->ip.reass.range_first)
1050 fragment_last - candidate_vnb->ip.reass.range_first + 1;
1051 if (overlap < ip4_full_reass_buffer_get_data_len (candidate_b))
1053 candidate_vnb->ip.reass.range_first += overlap;
1054 if (reass->data_len < overlap)
1056 return IP4_REASS_RC_INTERNAL_ERROR;
1058 reass->data_len -= overlap;
1059 if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
1061 ip4_full_reass_add_trace (vm, node, reass,
1063 RANGE_SHRINK, 0, ~0);
1065 rc = ip4_full_reass_insert_range_in_chain (
1066 vm, reass, prev_range_bi, *bi0);
1067 if (IP4_REASS_RC_OK != rc)
1075 discard_candidate = 1;
1078 else if (fragment_last > candidate_vnb->ip.reass.range_last)
1081 candidate_vnb->ip.reass.range_last - fragment_first + 1;
1082 if (overlap < ip4_full_reass_buffer_get_data_len (candidate_b))
1084 fvnb->ip.reass.range_first += overlap;
1085 if (~0 != candidate_vnb->ip.reass.next_range_bi)
1087 prev_range_bi = candidate_range_bi;
1088 candidate_range_bi =
1089 candidate_vnb->ip.reass.next_range_bi;
1094 // special case - last range discarded
1095 rc = ip4_full_reass_insert_range_in_chain (
1096 vm, reass, candidate_range_bi, *bi0);
1097 if (IP4_REASS_RC_OK != rc)
1106 discard_candidate = 1;
1111 discard_candidate = 1;
1113 if (discard_candidate)
1115 u32 next_range_bi = candidate_vnb->ip.reass.next_range_bi;
1116 // discard candidate range, probe next range
1117 rc = ip4_full_reass_remove_range_from_chain (
1118 vm, node, reass, prev_range_bi, candidate_range_bi);
1119 if (IP4_REASS_RC_OK != rc)
1123 if (~0 != next_range_bi)
1125 candidate_range_bi = next_range_bi;
1130 // special case - last range discarded
1131 rc = ip4_full_reass_insert_range_in_chain (
1132 vm, reass, prev_range_bi, *bi0);
1133 if (IP4_REASS_RC_OK != rc)
1143 ++reass->fragments_n;
1146 if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
1148 ip4_full_reass_add_trace (vm, node, reass, *bi0, RANGE_NEW, 0, ~0);
1151 if (~0 != reass->last_packet_octet &&
1152 reass->data_len == reass->last_packet_octet + 1)
1154 *handoff_thread_idx = reass->sendout_thread_index;
1156 reass->memory_owner_thread_index != reass->sendout_thread_index;
1158 ip4_full_reass_finalize (vm, node, rm, rt, reass, bi0, next0, error0,
1160 if (IP4_REASS_RC_OK == rc && handoff)
1162 rc = IP4_REASS_RC_HANDOFF;
1170 if (reass->fragments_n > rm->max_reass_len)
1172 rc = IP4_REASS_RC_TOO_MANY_FRAGMENTS;
1177 *next0 = IP4_FULL_REASS_NEXT_DROP;
1178 *error0 = IP4_ERROR_REASS_DUPLICATE_FRAGMENT;
1185 ip4_full_reass_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1186 vlib_frame_t *frame, ip4_full_reass_node_type_t type,
1189 u32 *from = vlib_frame_vector_args (frame);
1190 u32 n_left_from, n_left_to_next, *to_next, next_index;
1191 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1192 ip4_full_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
1193 clib_spinlock_lock (&rt->lock);
1195 n_left_from = frame->n_vectors;
1196 next_index = node->cached_next_index;
1197 while (n_left_from > 0)
1199 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1201 while (n_left_from > 0 && n_left_to_next > 0)
1206 u32 error0 = IP4_ERROR_NONE;
1209 b0 = vlib_get_buffer (vm, bi0);
1211 ip4_header_t *ip0 = vlib_buffer_get_current (b0);
1212 if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
1214 // this is a whole packet - no fragmentation
1217 next0 = IP4_FULL_REASS_NEXT_INPUT;
1221 next0 = vnet_buffer (b0)->ip.reass.next_index;
1223 ip4_full_reass_add_trace (vm, node, NULL, bi0, PASSTHROUGH, 0,
1225 goto packet_enqueue;
1228 if (is_local && !rm->is_local_reass_enabled)
1230 next0 = IP4_FULL_REASS_NEXT_DROP;
1231 goto packet_enqueue;
1234 const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
1235 const u32 fragment_length =
1236 clib_net_to_host_u16 (ip0->length) - ip4_header_bytes (ip0);
1237 const u32 fragment_last = fragment_first + fragment_length - 1;
1239 /* Keep track of received fragments */
1240 vlib_node_increment_counter (vm, node->node_index,
1241 IP4_ERROR_REASS_FRAGMENTS_RCVD, 1);
1243 if (fragment_first > fragment_last ||
1244 fragment_first + fragment_length > UINT16_MAX - 20 ||
1245 (fragment_length < 8 && // 8 is minimum frag length per RFC 791
1246 ip4_get_fragment_more (ip0)))
1248 next0 = IP4_FULL_REASS_NEXT_DROP;
1249 error0 = IP4_ERROR_REASS_MALFORMED_PACKET;
1250 goto packet_enqueue;
1252 ip4_full_reass_kv_t kv;
1256 (u64) vec_elt (ip4_main.fib_index_by_sw_if_index,
1257 vnet_buffer (b0)->sw_if_index[VLIB_RX]) |
1258 (u64) ip0->src_address.as_u32 << 32;
1260 (u64) ip0->dst_address.
1261 as_u32 | (u64) ip0->fragment_id << 32 | (u64) ip0->protocol << 48;
1263 ip4_full_reass_t *reass = ip4_full_reass_find_or_create (
1264 vm, node, rm, rt, &kv, &do_handoff, &n_left_to_next, &to_next);
1268 const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
1269 if (0 == fragment_first)
1271 reass->sendout_thread_index = vm->thread_index;
1275 if (PREDICT_FALSE (do_handoff))
1277 next0 = IP4_FULL_REASS_NEXT_HANDOFF;
1278 vnet_buffer (b0)->ip.reass.owner_thread_index =
1279 kv.v.memory_owner_thread_index;
1283 u32 handoff_thread_idx;
1285 switch (ip4_full_reass_update
1286 (vm, node, rm, rt, reass, &bi0, &next0,
1287 &error0, CUSTOM == type, &handoff_thread_idx))
1289 case IP4_REASS_RC_OK:
1290 /* nothing to do here */
1292 case IP4_REASS_RC_HANDOFF:
1293 next0 = IP4_FULL_REASS_NEXT_HANDOFF;
1294 b0 = vlib_get_buffer (vm, bi0);
1295 vnet_buffer (b0)->ip.reass.owner_thread_index =
1298 case IP4_REASS_RC_TOO_MANY_FRAGMENTS:
1299 counter = IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG;
1301 case IP4_REASS_RC_NO_BUF:
1302 counter = IP4_ERROR_REASS_NO_BUF;
1304 case IP4_REASS_RC_INTERNAL_ERROR:
1305 counter = IP4_ERROR_REASS_INTERNAL_ERROR;
1306 /* Sanitization is needed in internal error cases only, as
1307 * the incoming packet is already dropped in other cases,
1308 * also adding bi0 back to the reassembly list, fixes the
1309 * leaking of buffers during internal errors.
1311 * Also it doesnt make sense to send these buffers custom
1312 * app, these fragments are with internal errors */
1313 sanitize_reass_buffers_add_missing (vm, reass, &bi0);
1314 reass->error_next_index = ~0;
1320 vlib_node_increment_counter (vm, node->node_index, counter,
1322 ip4_full_reass_drop_all (vm, node, reass, &n_left_to_next,
1324 ip4_full_reass_free (rm, rt, reass);
1330 next0 = IP4_FULL_REASS_NEXT_DROP;
1331 error0 = IP4_ERROR_REASS_LIMIT_REACHED;
1341 n_left_to_next -= 1;
1343 /* bi0 might have been updated by reass_finalize, reload */
1344 b0 = vlib_get_buffer (vm, bi0);
1345 if (IP4_ERROR_NONE != error0)
1347 b0->error = node->errors[error0];
1350 if (next0 == IP4_FULL_REASS_NEXT_HANDOFF)
1352 if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1354 ip4_full_reass_add_trace (
1355 vm, node, NULL, bi0, HANDOFF, 0,
1356 vnet_buffer (b0)->ip.reass.owner_thread_index);
1359 else if (FEATURE == type && IP4_ERROR_NONE == error0)
1361 vnet_feature_next (&next0, b0);
1364 /* Increment the counter to-custom-app also as this fragment is
1365 * also going to application */
1368 vlib_node_increment_counter (
1369 vm, node->node_index, IP4_ERROR_REASS_TO_CUSTOM_APP, 1);
1372 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1373 to_next, n_left_to_next,
1375 IP4_REASS_DEBUG_BUFFER (bi0, enqueue_next);
1383 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1386 clib_spinlock_unlock (&rt->lock);
1387 return frame->n_vectors;
1390 static char *ip4_full_reass_error_strings[] = {
1391 #define _(sym, string) string,
1396 VLIB_NODE_FN (ip4_full_reass_node) (vlib_main_t * vm,
1397 vlib_node_runtime_t * node,
1398 vlib_frame_t * frame)
1400 return ip4_full_reass_inline (vm, node, frame, NORMAL, false /* is_local */);
1403 VLIB_REGISTER_NODE (ip4_full_reass_node) = {
1404 .name = "ip4-full-reassembly",
1405 .vector_size = sizeof (u32),
1406 .format_trace = format_ip4_full_reass_trace,
1407 .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1408 .error_strings = ip4_full_reass_error_strings,
1409 .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1412 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1413 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1414 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reassembly-handoff",
1419 VLIB_NODE_FN (ip4_local_full_reass_node)
1420 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1422 return ip4_full_reass_inline (vm, node, frame, NORMAL, true /* is_local */);
1425 VLIB_REGISTER_NODE (ip4_local_full_reass_node) = {
1426 .name = "ip4-local-full-reassembly",
1427 .vector_size = sizeof (u32),
1428 .format_trace = format_ip4_full_reass_trace,
1429 .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1430 .error_strings = ip4_full_reass_error_strings,
1431 .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1434 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1435 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1436 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-local-full-reassembly-handoff",
1441 VLIB_NODE_FN (ip4_full_reass_node_feature) (vlib_main_t * vm,
1442 vlib_node_runtime_t * node,
1443 vlib_frame_t * frame)
1445 return ip4_full_reass_inline (vm, node, frame, FEATURE,
1446 false /* is_local */);
1449 VLIB_REGISTER_NODE (ip4_full_reass_node_feature) = {
1450 .name = "ip4-full-reassembly-feature",
1451 .vector_size = sizeof (u32),
1452 .format_trace = format_ip4_full_reass_trace,
1453 .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1454 .error_strings = ip4_full_reass_error_strings,
1455 .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1458 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1459 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1460 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reass-feature-hoff",
1464 VNET_FEATURE_INIT (ip4_full_reass_feature, static) = {
1465 .arc_name = "ip4-unicast",
1466 .node_name = "ip4-full-reassembly-feature",
1467 .runs_before = VNET_FEATURES ("ip4-lookup",
1468 "ipsec4-input-feature"),
1472 VLIB_NODE_FN (ip4_full_reass_node_custom) (vlib_main_t * vm,
1473 vlib_node_runtime_t * node,
1474 vlib_frame_t * frame)
1476 return ip4_full_reass_inline (vm, node, frame, CUSTOM, false /* is_local */);
1479 VLIB_REGISTER_NODE (ip4_full_reass_node_custom) = {
1480 .name = "ip4-full-reassembly-custom",
1481 .vector_size = sizeof (u32),
1482 .format_trace = format_ip4_full_reass_trace,
1483 .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1484 .error_strings = ip4_full_reass_error_strings,
1485 .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1488 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1489 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1490 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reass-custom-hoff",
1494 VNET_FEATURE_INIT (ip4_full_reass_custom, static) = {
1495 .arc_name = "ip4-unicast",
1496 .node_name = "ip4-full-reassembly-feature",
1497 .runs_before = VNET_FEATURES ("ip4-lookup",
1498 "ipsec4-input-feature"),
1503 #ifndef CLIB_MARCH_VARIANT
1505 ip4_full_reass_custom_register_next_node (uword node_index)
1507 return vlib_node_add_next (vlib_get_main (),
1508 ip4_full_reass_node_custom.index, node_index);
1512 ip4_full_reass_get_nbuckets ()
1514 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1518 /* need more mem with more workers */
1519 nbuckets = (u32) (rm->max_reass_n * (vlib_num_workers () + 1) /
1520 IP4_REASS_HT_LOAD_FACTOR);
1522 for (i = 0; i < 31; i++)
1523 if ((1 << i) >= nbuckets)
1529 #endif /* CLIB_MARCH_VARIANT */
1533 IP4_EVENT_CONFIG_CHANGED = 1,
1534 } ip4_full_reass_event_t;
1539 clib_bihash_16_8_t *new_hash;
1540 } ip4_rehash_cb_ctx;
1542 #ifndef CLIB_MARCH_VARIANT
1544 ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx)
1546 ip4_rehash_cb_ctx *ctx = _ctx;
1547 if (clib_bihash_add_del_16_8 (ctx->new_hash, kv, 1))
1551 return (BIHASH_WALK_CONTINUE);
1555 ip4_full_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
1556 u32 max_reassembly_length,
1557 u32 expire_walk_interval_ms)
1559 ip4_full_reass_main.timeout_ms = timeout_ms;
1560 ip4_full_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
1561 ip4_full_reass_main.max_reass_n = max_reassemblies;
1562 ip4_full_reass_main.max_reass_len = max_reassembly_length;
1563 ip4_full_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
1567 ip4_full_reass_set (u32 timeout_ms, u32 max_reassemblies,
1568 u32 max_reassembly_length, u32 expire_walk_interval_ms)
1570 u32 old_nbuckets = ip4_full_reass_get_nbuckets ();
1571 ip4_full_reass_set_params (timeout_ms, max_reassemblies,
1572 max_reassembly_length, expire_walk_interval_ms);
1573 vlib_process_signal_event (ip4_full_reass_main.vlib_main,
1574 ip4_full_reass_main.ip4_full_reass_expire_node_idx,
1575 IP4_EVENT_CONFIG_CHANGED, 0);
1576 u32 new_nbuckets = ip4_full_reass_get_nbuckets ();
1577 if (ip4_full_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
1579 clib_bihash_16_8_t new_hash;
1580 clib_memset (&new_hash, 0, sizeof (new_hash));
1581 ip4_rehash_cb_ctx ctx;
1583 ctx.new_hash = &new_hash;
1584 clib_bihash_init_16_8 (&new_hash, "ip4-dr", new_nbuckets,
1585 new_nbuckets * 1024);
1586 clib_bihash_foreach_key_value_pair_16_8 (&ip4_full_reass_main.hash,
1587 ip4_rehash_cb, &ctx);
1590 clib_bihash_free_16_8 (&new_hash);
1595 clib_bihash_free_16_8 (&ip4_full_reass_main.hash);
1596 clib_memcpy_fast (&ip4_full_reass_main.hash, &new_hash,
1597 sizeof (ip4_full_reass_main.hash));
1598 clib_bihash_copied (&ip4_full_reass_main.hash, &new_hash);
1605 ip4_full_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
1606 u32 * max_reassembly_length,
1607 u32 * expire_walk_interval_ms)
1609 *timeout_ms = ip4_full_reass_main.timeout_ms;
1610 *max_reassemblies = ip4_full_reass_main.max_reass_n;
1611 *max_reassembly_length = ip4_full_reass_main.max_reass_len;
1612 *expire_walk_interval_ms = ip4_full_reass_main.expire_walk_interval_ms;
1616 static clib_error_t *
1617 ip4_full_reass_init_function (vlib_main_t * vm)
1619 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1620 clib_error_t *error = 0;
1626 vec_validate (rm->per_thread_data, vlib_num_workers ());
1627 ip4_full_reass_per_thread_t *rt;
1628 vec_foreach (rt, rm->per_thread_data)
1630 clib_spinlock_init (&rt->lock);
1631 pool_alloc (rt->pool, rm->max_reass_n);
1634 node = vlib_get_node_by_name (vm, (u8 *) "ip4-full-reassembly-expire-walk");
1636 rm->ip4_full_reass_expire_node_idx = node->index;
1638 ip4_full_reass_set_params (IP4_REASS_TIMEOUT_DEFAULT_MS,
1639 IP4_REASS_MAX_REASSEMBLIES_DEFAULT,
1640 IP4_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
1641 IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
1643 nbuckets = ip4_full_reass_get_nbuckets ();
1644 clib_bihash_init_16_8 (&rm->hash, "ip4-dr", nbuckets, nbuckets * 1024);
1646 rm->fq_index = vlib_frame_queue_main_init (ip4_full_reass_node.index, 0);
1647 rm->fq_local_index =
1648 vlib_frame_queue_main_init (ip4_local_full_reass_node.index, 0);
1649 rm->fq_feature_index =
1650 vlib_frame_queue_main_init (ip4_full_reass_node_feature.index, 0);
1651 rm->fq_custom_index =
1652 vlib_frame_queue_main_init (ip4_full_reass_node_custom.index, 0);
1654 rm->feature_use_refcount_per_intf = NULL;
1655 rm->is_local_reass_enabled = 1;
1660 VLIB_INIT_FUNCTION (ip4_full_reass_init_function);
1661 #endif /* CLIB_MARCH_VARIANT */
1664 ip4_full_reass_walk_expired (vlib_main_t *vm, vlib_node_runtime_t *node,
1665 CLIB_UNUSED (vlib_frame_t *f))
1667 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1668 uword event_type, *event_data = 0;
1672 vlib_process_wait_for_event_or_clock (vm,
1674 rm->expire_walk_interval_ms /
1675 (f64) MSEC_PER_SEC);
1676 event_type = vlib_process_get_events (vm, &event_data);
1681 /* no events => timeout */
1683 case IP4_EVENT_CONFIG_CHANGED:
1684 /* nothing to do here */
1687 clib_warning ("BUG: event type 0x%wx", event_type);
1690 f64 now = vlib_time_now (vm);
1692 ip4_full_reass_t *reass;
1693 int *pool_indexes_to_free = NULL;
1695 uword thread_index = 0;
1697 const uword nthreads = vlib_num_workers () + 1;
1698 u32 n_left_to_next, *to_next;
1700 for (thread_index = 0; thread_index < nthreads; ++thread_index)
1702 ip4_full_reass_per_thread_t *rt =
1703 &rm->per_thread_data[thread_index];
1704 clib_spinlock_lock (&rt->lock);
1706 vec_reset_length (pool_indexes_to_free);
1708 /* Pace the number of timeouts handled per thread,to avoid barrier
1709 * sync issues in real world scenarios */
1711 u32 beg = rt->last_id;
1712 /* to ensure we walk at least once per sec per context */
1714 beg + (IP4_REASS_MAX_REASSEMBLIES_DEFAULT *
1715 IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS / MSEC_PER_SEC +
1717 if (end > vec_len (rt->pool))
1719 end = vec_len (rt->pool);
1727 pool_foreach_stepping_index (index, beg, end, rt->pool)
1729 reass = pool_elt_at_index (rt->pool, index);
1730 if (now > reass->last_heard + rm->timeout)
1732 vec_add1 (pool_indexes_to_free, index);
1736 if (vec_len (pool_indexes_to_free))
1737 vlib_node_increment_counter (vm, node->node_index,
1738 IP4_ERROR_REASS_TIMEOUT,
1739 vec_len (pool_indexes_to_free));
1741 vec_foreach (i, pool_indexes_to_free)
1743 ip4_full_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1744 ip4_full_reass_drop_all (vm, node, reass, &n_left_to_next,
1746 ip4_full_reass_free (rm, rt, reass);
1749 clib_spinlock_unlock (&rt->lock);
1752 vec_free (pool_indexes_to_free);
1755 vec_set_len (event_data, 0);
1762 VLIB_REGISTER_NODE (ip4_full_reass_expire_node) = {
1763 .function = ip4_full_reass_walk_expired,
1764 .type = VLIB_NODE_TYPE_PROCESS,
1765 .name = "ip4-full-reassembly-expire-walk",
1766 .format_trace = format_ip4_full_reass_trace,
1767 .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1768 .error_strings = ip4_full_reass_error_strings,
1773 format_ip4_full_reass_key (u8 * s, va_list * args)
1775 ip4_full_reass_key_t *key = va_arg (*args, ip4_full_reass_key_t *);
1778 "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1779 key->xx_id, format_ip4_address, &key->src, format_ip4_address,
1780 &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1785 format_ip4_reass (u8 * s, va_list * args)
1787 vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1788 ip4_full_reass_t *reass = va_arg (*args, ip4_full_reass_t *);
1790 s = format (s, "ID: %lu, key: %U\n first_bi: %u, data_len: %u, "
1791 "last_packet_octet: %u, trace_op_counter: %u\n",
1792 reass->id, format_ip4_full_reass_key, &reass->key,
1793 reass->first_bi, reass->data_len,
1794 reass->last_packet_octet, reass->trace_op_counter);
1796 u32 bi = reass->first_bi;
1800 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1801 vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1804 " #%03u: range: [%u, %u], bi: %u, off: %d, len: %u, "
1805 "fragment[%u, %u]\n", counter, vnb->ip.reass.range_first,
1806 vnb->ip.reass.range_last, bi,
1807 ip4_full_reass_buffer_get_data_offset (b),
1808 ip4_full_reass_buffer_get_data_len (b),
1809 vnb->ip.reass.fragment_first, vnb->ip.reass.fragment_last);
1810 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1812 bi = b->next_buffer;
1822 static clib_error_t *
1823 show_ip4_reass (vlib_main_t * vm,
1824 unformat_input_t * input,
1825 CLIB_UNUSED (vlib_cli_command_t * lmd))
1827 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1829 vlib_cli_output (vm, "---------------------");
1830 vlib_cli_output (vm, "IP4 reassembly status");
1831 vlib_cli_output (vm, "---------------------");
1832 bool details = false;
1833 if (unformat (input, "details"))
1838 u32 sum_reass_n = 0;
1839 ip4_full_reass_t *reass;
1841 const uword nthreads = vlib_num_workers () + 1;
1842 for (thread_index = 0; thread_index < nthreads; ++thread_index)
1844 ip4_full_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1845 clib_spinlock_lock (&rt->lock);
1848 pool_foreach (reass, rt->pool) {
1849 vlib_cli_output (vm, "%U", format_ip4_reass, vm, reass);
1852 sum_reass_n += rt->reass_n;
1853 clib_spinlock_unlock (&rt->lock);
1855 vlib_cli_output (vm, "---------------------");
1856 vlib_cli_output (vm, "Current full IP4 reassemblies count: %lu\n",
1857 (long unsigned) sum_reass_n);
1858 vlib_cli_output (vm,
1859 "Maximum configured concurrent full IP4 reassemblies per worker-thread: %lu\n",
1860 (long unsigned) rm->max_reass_n);
1861 vlib_cli_output (vm,
1862 "Maximum configured amount of fragments "
1863 "per full IP4 reassembly: %lu\n",
1864 (long unsigned) rm->max_reass_len);
1865 vlib_cli_output (vm,
1866 "Maximum configured full IP4 reassembly timeout: %lums\n",
1867 (long unsigned) rm->timeout_ms);
1868 vlib_cli_output (vm,
1869 "Maximum configured full IP4 reassembly expire walk interval: %lums\n",
1870 (long unsigned) rm->expire_walk_interval_ms);
1874 VLIB_CLI_COMMAND (show_ip4_full_reass_cmd, static) = {
1875 .path = "show ip4-full-reassembly",
1876 .short_help = "show ip4-full-reassembly [details]",
1877 .function = show_ip4_reass,
1880 #ifndef CLIB_MARCH_VARIANT
1882 ip4_full_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1884 return vnet_feature_enable_disable ("ip4-unicast",
1885 "ip4-full-reassembly-feature",
1886 sw_if_index, enable_disable, 0, 0);
1888 #endif /* CLIB_MARCH_VARIANT */
1891 #define foreach_ip4_full_reass_handoff_error \
1892 _(CONGESTION_DROP, "congestion drop")
1897 #define _(sym,str) IP4_FULL_REASS_HANDOFF_ERROR_##sym,
1898 foreach_ip4_full_reass_handoff_error
1900 IP4_FULL_REASS_HANDOFF_N_ERROR,
1901 } ip4_full_reass_handoff_error_t;
1903 static char *ip4_full_reass_handoff_error_strings[] = {
1904 #define _(sym,string) string,
1905 foreach_ip4_full_reass_handoff_error
1911 u32 next_worker_index;
1912 } ip4_full_reass_handoff_trace_t;
1915 format_ip4_full_reass_handoff_trace (u8 * s, va_list * args)
1917 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1918 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1919 ip4_full_reass_handoff_trace_t *t =
1920 va_arg (*args, ip4_full_reass_handoff_trace_t *);
1923 format (s, "ip4-full-reassembly-handoff: next-worker %d",
1924 t->next_worker_index);
1930 ip4_full_reass_handoff_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1931 vlib_frame_t *frame,
1932 ip4_full_reass_node_type_t type,
1935 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1937 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1938 u32 n_enq, n_left_from, *from;
1939 u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1942 from = vlib_frame_vector_args (frame);
1943 n_left_from = frame->n_vectors;
1944 vlib_get_buffers (vm, from, bufs, n_left_from);
1947 ti = thread_indices;
1954 fq_index = rm->fq_local_index;
1958 fq_index = rm->fq_index;
1962 fq_index = rm->fq_feature_index;
1965 fq_index = rm->fq_custom_index;
1968 clib_warning ("Unexpected `type' (%d)!", type);
1971 while (n_left_from > 0)
1973 ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1976 ((node->flags & VLIB_NODE_FLAG_TRACE)
1977 && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1979 ip4_full_reass_handoff_trace_t *t =
1980 vlib_add_trace (vm, node, b[0], sizeof (*t));
1981 t->next_worker_index = ti[0];
1988 n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1989 thread_indices, frame->n_vectors, 1);
1991 if (n_enq < frame->n_vectors)
1992 vlib_node_increment_counter (vm, node->node_index,
1993 IP4_FULL_REASS_HANDOFF_ERROR_CONGESTION_DROP,
1994 frame->n_vectors - n_enq);
1995 return frame->n_vectors;
1998 VLIB_NODE_FN (ip4_full_reass_handoff_node) (vlib_main_t * vm,
1999 vlib_node_runtime_t * node,
2000 vlib_frame_t * frame)
2002 return ip4_full_reass_handoff_node_inline (vm, node, frame, NORMAL,
2003 false /* is_local */);
2007 VLIB_REGISTER_NODE (ip4_full_reass_handoff_node) = {
2008 .name = "ip4-full-reassembly-handoff",
2009 .vector_size = sizeof (u32),
2010 .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
2011 .error_strings = ip4_full_reass_handoff_error_strings,
2012 .format_trace = format_ip4_full_reass_handoff_trace,
2021 VLIB_NODE_FN (ip4_local_full_reass_handoff_node)
2022 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
2024 return ip4_full_reass_handoff_node_inline (vm, node, frame, NORMAL,
2025 true /* is_local */);
2028 VLIB_REGISTER_NODE (ip4_local_full_reass_handoff_node) = {
2029 .name = "ip4-local-full-reassembly-handoff",
2030 .vector_size = sizeof (u32),
2031 .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
2032 .error_strings = ip4_full_reass_handoff_error_strings,
2033 .format_trace = format_ip4_full_reass_handoff_trace,
2042 VLIB_NODE_FN (ip4_full_reass_feature_handoff_node) (vlib_main_t * vm,
2043 vlib_node_runtime_t *
2045 vlib_frame_t * frame)
2047 return ip4_full_reass_handoff_node_inline (vm, node, frame, FEATURE,
2048 false /* is_local */);
2051 VLIB_REGISTER_NODE (ip4_full_reass_feature_handoff_node) = {
2052 .name = "ip4-full-reass-feature-hoff",
2053 .vector_size = sizeof (u32),
2054 .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
2055 .error_strings = ip4_full_reass_handoff_error_strings,
2056 .format_trace = format_ip4_full_reass_handoff_trace,
2065 VLIB_NODE_FN (ip4_full_reass_custom_handoff_node) (vlib_main_t * vm,
2066 vlib_node_runtime_t *
2068 vlib_frame_t * frame)
2070 return ip4_full_reass_handoff_node_inline (vm, node, frame, CUSTOM,
2071 false /* is_local */);
2074 VLIB_REGISTER_NODE (ip4_full_reass_custom_handoff_node) = {
2075 .name = "ip4-full-reass-custom-hoff",
2076 .vector_size = sizeof (u32),
2077 .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
2078 .error_strings = ip4_full_reass_handoff_error_strings,
2079 .format_trace = format_ip4_full_reass_handoff_trace,
2088 #ifndef CLIB_MARCH_VARIANT
2090 ip4_full_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
2092 ip4_full_reass_main_t *rm = &ip4_full_reass_main;
2093 vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
2096 if (!rm->feature_use_refcount_per_intf[sw_if_index])
2098 ++rm->feature_use_refcount_per_intf[sw_if_index];
2099 return vnet_feature_enable_disable ("ip4-unicast",
2100 "ip4-full-reassembly-feature",
2101 sw_if_index, 1, 0, 0);
2103 ++rm->feature_use_refcount_per_intf[sw_if_index];
2107 --rm->feature_use_refcount_per_intf[sw_if_index];
2108 if (!rm->feature_use_refcount_per_intf[sw_if_index])
2109 return vnet_feature_enable_disable ("ip4-unicast",
2110 "ip4-full-reassembly-feature",
2111 sw_if_index, 0, 0, 0);
2117 ip4_local_full_reass_enable_disable (int enable)
2121 ip4_full_reass_main.is_local_reass_enabled = 1;
2125 ip4_full_reass_main.is_local_reass_enabled = 0;
2130 ip4_local_full_reass_enabled ()
2132 return ip4_full_reass_main.is_local_reass_enabled;
2138 * fd.io coding-style-patch-verification: ON
2141 * eval: (c-set-style "gnu")