src/vnet/ip/reass/ip4_full_reass.c

   1 /*
   2  * Copyright (c) 2017 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 /**
  17  * @file
  18  * @brief IPv4 Full Reassembly.
  19  *
  20  * This file contains the source code for IPv4 full reassembly.
  21  */
  22
  23 #include <vppinfra/vec.h>
  24 #include <vnet/vnet.h>
  25 #include <vnet/ip/ip.h>
  26 #include <vppinfra/fifo.h>
  27 #include <vppinfra/bihash_16_8.h>
  28 #include <vnet/ip/reass/ip4_full_reass.h>
  29 #include <stddef.h>
  30
  31 #define MSEC_PER_SEC 1000
  32 #define IP4_REASS_TIMEOUT_DEFAULT_MS 100
  33 #define IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000 // 10 seconds default
  34 #define IP4_REASS_MAX_REASSEMBLIES_DEFAULT 1024
  35 #define IP4_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
  36 #define IP4_REASS_HT_LOAD_FACTOR (0.75)
  37
  38 #define IP4_REASS_DEBUG_BUFFERS 0
  39 #if IP4_REASS_DEBUG_BUFFERS
  40 #define IP4_REASS_DEBUG_BUFFER(bi, what)             \
  41   do                                                 \
  42     {                                                \
  43       u32 _bi = bi;                                  \
  44       printf (#what "buffer %u", _bi);               \
  45       vlib_buffer_t *_b = vlib_get_buffer (vm, _bi); \
  46       while (_b->flags & VLIB_BUFFER_NEXT_PRESENT)   \
  47         {                                            \
  48           _bi = _b->next_buffer;                     \
  49           printf ("[%u]", _bi);                      \
  50           _b = vlib_get_buffer (vm, _bi);            \
  51         }                                            \
  52       printf ("\n");                                 \
  53       fflush (stdout);                               \
  54     }                                                \
  55   while (0)
  56 #else
  57 #define IP4_REASS_DEBUG_BUFFER(...)
  58 #endif
  59
  60 typedef enum
  61 {
  62   IP4_REASS_RC_OK,
  63   IP4_REASS_RC_TOO_MANY_FRAGMENTS,
  64   IP4_REASS_RC_INTERNAL_ERROR,
  65   IP4_REASS_RC_NO_BUF,
  66   IP4_REASS_RC_HANDOFF,
  67 } ip4_full_reass_rc_t;
  68
  69 typedef struct
  70 {
  71   union
  72   {
  73     struct
  74     {
  75       u32 xx_id;
  76       ip4_address_t src;
  77       ip4_address_t dst;
  78       u16 frag_id;
  79       u8 proto;
  80       u8 unused;
  81     };
  82     u64 as_u64[2];
  83   };
  84 } ip4_full_reass_key_t;
  85
  86 typedef union
  87 {
  88   struct
  89   {
  90     u32 reass_index;
  91     u32 memory_owner_thread_index;
  92   };
  93   u64 as_u64;
  94 } ip4_full_reass_val_t;
  95
  96 typedef union
  97 {
  98   struct
  99   {
 100     ip4_full_reass_key_t k;
 101     ip4_full_reass_val_t v;
 102   };
 103   clib_bihash_kv_16_8_t kv;
 104 } ip4_full_reass_kv_t;
 105
 106 always_inline u32
 107 ip4_full_reass_buffer_get_data_offset (vlib_buffer_t * b)
 108 {
 109   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 110   return vnb->ip.reass.range_first - vnb->ip.reass.fragment_first;
 111 }
 112
 113 always_inline u16
 114 ip4_full_reass_buffer_get_data_len (vlib_buffer_t * b)
 115 {
 116   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 117   return clib_min (vnb->ip.reass.range_last, vnb->ip.reass.fragment_last) -
 118     (vnb->ip.reass.fragment_first +
 119      ip4_full_reass_buffer_get_data_offset (b)) + 1;
 120 }
 121
 122 typedef struct
 123 {
 124   // hash table key
 125   ip4_full_reass_key_t key;
 126   // time when last packet was received
 127   f64 last_heard;
 128   // internal id of this reassembly
 129   u64 id;
 130   // buffer index of first buffer in this reassembly context
 131   u32 first_bi;
 132   // last octet of packet, ~0 until fragment without more_fragments arrives
 133   u32 last_packet_octet;
 134   // length of data collected so far
 135   u32 data_len;
 136   // trace operation counter
 137   u32 trace_op_counter;
 138   // next index - used by non-feature node
 139   u32 next_index;
 140   // error next index - used by custom apps (~0 if not used)
 141   u32 error_next_index;
 142   // minimum fragment length for this reassembly - used to estimate MTU
 143   u16 min_fragment_length;
 144   // number of fragments in this reassembly
 145   u32 fragments_n;
 146   // thread owning memory for this context (whose pool contains this ctx)
 147   u32 memory_owner_thread_index;
 148   // thread which received fragment with offset 0 and which sends out the
 149   // completed reassembly
 150   u32 sendout_thread_index;
 151 } ip4_full_reass_t;
 152
 153 typedef struct
 154 {
 155   ip4_full_reass_t *pool;
 156   u32 reass_n;
 157   u32 id_counter;
 158   clib_spinlock_t lock;
 159 } ip4_full_reass_per_thread_t;
 160
 161 typedef struct
 162 {
 163   // IPv4 config
 164   u32 timeout_ms;
 165   f64 timeout;
 166   u32 expire_walk_interval_ms;
 167   // maximum number of fragments in one reassembly
 168   u32 max_reass_len;
 169   // maximum number of reassemblies
 170   u32 max_reass_n;
 171
 172   // IPv4 runtime
 173   clib_bihash_16_8_t hash;
 174   // per-thread data
 175   ip4_full_reass_per_thread_t *per_thread_data;
 176
 177   // convenience
 178   vlib_main_t *vlib_main;
 179
 180   // node index of ip4-drop node
 181   u32 ip4_drop_idx;
 182   u32 ip4_full_reass_expire_node_idx;
 183
 184   /** Worker handoff */
 185   u32 fq_index;
 186   u32 fq_local_index;
 187   u32 fq_feature_index;
 188   u32 fq_custom_index;
 189
 190   // reference count for enabling/disabling feature - per interface
 191   u32 *feature_use_refcount_per_intf;
 192
 193   // whether local fragmented packets are reassembled or not
 194   int is_local_reass_enabled;
 195 } ip4_full_reass_main_t;
 196
 197 extern ip4_full_reass_main_t ip4_full_reass_main;
 198
 199 #ifndef CLIB_MARCH_VARIANT
 200 ip4_full_reass_main_t ip4_full_reass_main;
 201 #endif /* CLIB_MARCH_VARIANT */
 202
 203 typedef enum
 204 {
 205   IP4_FULL_REASS_NEXT_INPUT,
 206   IP4_FULL_REASS_NEXT_DROP,
 207   IP4_FULL_REASS_NEXT_HANDOFF,
 208   IP4_FULL_REASS_N_NEXT,
 209 } ip4_full_reass_next_t;
 210
 211 typedef enum
 212 {
 213   NORMAL,
 214   FEATURE,
 215   CUSTOM
 216 } ip4_full_reass_node_type_t;
 217
 218 typedef enum
 219 {
 220   RANGE_NEW,
 221   RANGE_SHRINK,
 222   RANGE_DISCARD,
 223   RANGE_OVERLAP,
 224   FINALIZE,
 225   HANDOFF,
 226   PASSTHROUGH,
 227 } ip4_full_reass_trace_operation_e;
 228
 229 typedef struct
 230 {
 231   u16 range_first;
 232   u16 range_last;
 233   u32 range_bi;
 234   i32 data_offset;
 235   u32 data_len;
 236   u32 first_bi;
 237 } ip4_full_reass_range_trace_t;
 238
 239 typedef struct
 240 {
 241   ip4_full_reass_trace_operation_e action;
 242   u32 reass_id;
 243   ip4_full_reass_range_trace_t trace_range;
 244   u32 size_diff;
 245   u32 op_id;
 246   u32 thread_id;
 247   u32 thread_id_to;
 248   u32 fragment_first;
 249   u32 fragment_last;
 250   u32 total_data_len;
 251   bool is_after_handoff;
 252   ip4_header_t ip4_header;
 253 } ip4_full_reass_trace_t;
 254
 255 extern vlib_node_registration_t ip4_full_reass_node;
 256 extern vlib_node_registration_t ip4_full_reass_node_feature;
 257 extern vlib_node_registration_t ip4_full_reass_node_custom;
 258
 259 static void
 260 ip4_full_reass_trace_details (vlib_main_t * vm, u32 bi,
 261                               ip4_full_reass_range_trace_t * trace)
 262 {
 263   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 264   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 265   trace->range_first = vnb->ip.reass.range_first;
 266   trace->range_last = vnb->ip.reass.range_last;
 267   trace->data_offset = ip4_full_reass_buffer_get_data_offset (b);
 268   trace->data_len = ip4_full_reass_buffer_get_data_len (b);
 269   trace->range_bi = bi;
 270 }
 271
 272 static u8 *
 273 format_ip4_full_reass_range_trace (u8 * s, va_list * args)
 274 {
 275   ip4_full_reass_range_trace_t *trace =
 276     va_arg (*args, ip4_full_reass_range_trace_t *);
 277   s =
 278     format (s, "range: [%u, %u], off %d, len %u, bi %u", trace->range_first,
 279             trace->range_last, trace->data_offset, trace->data_len,
 280             trace->range_bi);
 281   return s;
 282 }
 283
 284 static u8 *
 285 format_ip4_full_reass_trace (u8 * s, va_list * args)
 286 {
 287   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 288   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 289   ip4_full_reass_trace_t *t = va_arg (*args, ip4_full_reass_trace_t *);
 290   u32 indent = 0;
 291   if (~0 != t->reass_id)
 292     {
 293       if (t->is_after_handoff)
 294         {
 295           s =
 296             format (s, "%U\n", format_ip4_header, &t->ip4_header,
 297                     sizeof (t->ip4_header));
 298           indent = 2;
 299         }
 300       s =
 301         format (s, "%Ureass id: %u, op id: %u, ", format_white_space, indent,
 302                 t->reass_id, t->op_id);
 303       indent = format_get_indent (s);
 304       s =
 305         format (s,
 306                 "first bi: %u, data len: %u, ip/fragment[%u, %u]",
 307                 t->trace_range.first_bi, t->total_data_len, t->fragment_first,
 308                 t->fragment_last);
 309     }
 310   switch (t->action)
 311     {
 312     case RANGE_SHRINK:
 313       s = format (s, "\n%Ushrink %U by %u", format_white_space, indent,
 314                   format_ip4_full_reass_range_trace, &t->trace_range,
 315                   t->size_diff);
 316       break;
 317     case RANGE_DISCARD:
 318       s = format (s, "\n%Udiscard %U", format_white_space, indent,
 319                   format_ip4_full_reass_range_trace, &t->trace_range);
 320       break;
 321     case RANGE_NEW:
 322       s = format (s, "\n%Unew %U", format_white_space, indent,
 323                   format_ip4_full_reass_range_trace, &t->trace_range);
 324       break;
 325     case RANGE_OVERLAP:
 326       s = format (s, "\n%Uoverlapping/ignored %U", format_white_space, indent,
 327                   format_ip4_full_reass_range_trace, &t->trace_range);
 328       break;
 329     case FINALIZE:
 330       s = format (s, "\n%Ufinalize reassembly", format_white_space, indent);
 331       break;
 332     case HANDOFF:
 333       s =
 334         format (s, "handoff from thread #%u to thread #%u", t->thread_id,
 335                 t->thread_id_to);
 336       break;
 337     case PASSTHROUGH:
 338       s = format (s, "passthrough - not a fragment");
 339       break;
 340     }
 341   return s;
 342 }
 343
 344 static void
 345 ip4_full_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
 346                           ip4_full_reass_t * reass, u32 bi,
 347                           ip4_full_reass_trace_operation_e action,
 348                           u32 size_diff, u32 thread_id_to)
 349 {
 350   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 351   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 352   if (pool_is_free_index
 353       (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b)))
 354     {
 355       // this buffer's trace is gone
 356       b->flags &= ~VLIB_BUFFER_IS_TRACED;
 357       return;
 358     }
 359   bool is_after_handoff = false;
 360   if (vlib_buffer_get_trace_thread (b) != vm->thread_index)
 361     {
 362       is_after_handoff = true;
 363     }
 364   ip4_full_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
 365   t->is_after_handoff = is_after_handoff;
 366   if (t->is_after_handoff)
 367     {
 368       clib_memcpy (&t->ip4_header, vlib_buffer_get_current (b),
 369                    clib_min (sizeof (t->ip4_header), b->current_length));
 370     }
 371   if (reass)
 372     {
 373       t->reass_id = reass->id;
 374       t->op_id = reass->trace_op_counter;
 375       t->trace_range.first_bi = reass->first_bi;
 376       t->total_data_len = reass->data_len;
 377       ++reass->trace_op_counter;
 378     }
 379   else
 380     {
 381       t->reass_id = ~0;
 382       t->op_id = 0;
 383       t->trace_range.first_bi = 0;
 384       t->total_data_len = 0;
 385     }
 386   t->action = action;
 387   ip4_full_reass_trace_details (vm, bi, &t->trace_range);
 388   t->size_diff = size_diff;
 389   t->thread_id = vm->thread_index;
 390   t->thread_id_to = thread_id_to;
 391   t->fragment_first = vnb->ip.reass.fragment_first;
 392   t->fragment_last = vnb->ip.reass.fragment_last;
 393 #if 0
 394   static u8 *s = NULL;
 395   s = format (s, "%U", format_ip4_full_reass_trace, NULL, NULL, t);
 396   printf ("%.*s\n", vec_len (s), s);
 397   fflush (stdout);
 398   vec_reset_length (s);
 399 #endif
 400 }
 401
 402 always_inline void
 403 ip4_full_reass_free_ctx (ip4_full_reass_per_thread_t * rt,
 404                          ip4_full_reass_t * reass)
 405 {
 406   pool_put (rt->pool, reass);
 407   --rt->reass_n;
 408 }
 409
 410 always_inline void
 411 ip4_full_reass_free (ip4_full_reass_main_t * rm,
 412                      ip4_full_reass_per_thread_t * rt,
 413                      ip4_full_reass_t * reass)
 414 {
 415   clib_bihash_kv_16_8_t kv;
 416   kv.key[0] = reass->key.as_u64[0];
 417   kv.key[1] = reass->key.as_u64[1];
 418   clib_bihash_add_del_16_8 (&rm->hash, &kv, 0);
 419   return ip4_full_reass_free_ctx (rt, reass);
 420 }
 421
 422 always_inline void
 423 ip4_full_reass_drop_all (vlib_main_t *vm, vlib_node_runtime_t *node,
 424                          ip4_full_reass_t *reass)
 425 {
 426   u32 range_bi = reass->first_bi;
 427   vlib_buffer_t *range_b;
 428   vnet_buffer_opaque_t *range_vnb;
 429   u32 *to_free = NULL;
 430   while (~0 != range_bi)
 431     {
 432       range_b = vlib_get_buffer (vm, range_bi);
 433       range_vnb = vnet_buffer (range_b);
 434       u32 bi = range_bi;
 435       while (~0 != bi)
 436         {
 437           vec_add1 (to_free, bi);
 438           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 439           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 440             {
 441               bi = b->next_buffer;
 442               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 443             }
 444           else
 445             {
 446               bi = ~0;
 447             }
 448         }
 449       range_bi = range_vnb->ip.reass.next_range_bi;
 450     }
 451   /* send to next_error_index */
 452   if (~0 != reass->error_next_index)
 453     {
 454       u32 n_left_to_next, *to_next, next_index;
 455
 456       next_index = reass->error_next_index;
 457       u32 bi = ~0;
 458
 459       while (vec_len (to_free) > 0)
 460         {
 461           vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 462
 463           while (vec_len (to_free) > 0 && n_left_to_next > 0)
 464             {
 465               bi = vec_pop (to_free);
 466
 467               if (~0 != bi)
 468                 {
 469                   to_next[0] = bi;
 470                   to_next += 1;
 471                   n_left_to_next -= 1;
 472                 }
 473             }
 474           vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 475         }
 476     }
 477   else
 478     {
 479       vlib_buffer_free (vm, to_free, vec_len (to_free));
 480     }
 481   vec_free (to_free);
 482 }
 483
 484 always_inline void
 485 ip4_full_reass_init (ip4_full_reass_t * reass)
 486 {
 487   reass->first_bi = ~0;
 488   reass->last_packet_octet = ~0;
 489   reass->data_len = 0;
 490   reass->next_index = ~0;
 491   reass->error_next_index = ~0;
 492 }
 493
 494 always_inline ip4_full_reass_t *
 495 ip4_full_reass_find_or_create (vlib_main_t * vm, vlib_node_runtime_t * node,
 496                                ip4_full_reass_main_t * rm,
 497                                ip4_full_reass_per_thread_t * rt,
 498                                ip4_full_reass_kv_t * kv, u8 * do_handoff)
 499 {
 500   ip4_full_reass_t *reass;
 501   f64 now;
 502
 503 again:
 504
 505   reass = NULL;
 506   now = vlib_time_now (vm);
 507   if (!clib_bihash_search_16_8 (&rm->hash, &kv->kv, &kv->kv))
 508     {
 509       if (vm->thread_index != kv->v.memory_owner_thread_index)
 510         {
 511           *do_handoff = 1;
 512           return NULL;
 513         }
 514       reass =
 515         pool_elt_at_index (rm->per_thread_data
 516                            [kv->v.memory_owner_thread_index].pool,
 517                            kv->v.reass_index);
 518
 519       if (now > reass->last_heard + rm->timeout)
 520         {
 521           ip4_full_reass_drop_all (vm, node, reass);
 522           ip4_full_reass_free (rm, rt, reass);
 523           reass = NULL;
 524         }
 525     }
 526
 527   if (reass)
 528     {
 529       reass->last_heard = now;
 530       return reass;
 531     }
 532
 533   if (rt->reass_n >= rm->max_reass_n)
 534     {
 535       reass = NULL;
 536       return reass;
 537     }
 538   else
 539     {
 540       pool_get (rt->pool, reass);
 541       clib_memset (reass, 0, sizeof (*reass));
 542       reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
 543       reass->memory_owner_thread_index = vm->thread_index;
 544       ++rt->id_counter;
 545       ip4_full_reass_init (reass);
 546       ++rt->reass_n;
 547     }
 548
 549   reass->key.as_u64[0] = kv->kv.key[0];
 550   reass->key.as_u64[1] = kv->kv.key[1];
 551   kv->v.reass_index = (reass - rt->pool);
 552   kv->v.memory_owner_thread_index = vm->thread_index;
 553   reass->last_heard = now;
 554
 555   int rv = clib_bihash_add_del_16_8 (&rm->hash, &kv->kv, 2);
 556   if (rv)
 557     {
 558       ip4_full_reass_free_ctx (rt, reass);
 559       reass = NULL;
 560       // if other worker created a context already work with the other copy
 561       if (-2 == rv)
 562         goto again;
 563     }
 564
 565   return reass;
 566 }
 567
 568 always_inline ip4_full_reass_rc_t
 569 ip4_full_reass_finalize (vlib_main_t * vm, vlib_node_runtime_t * node,
 570                          ip4_full_reass_main_t * rm,
 571                          ip4_full_reass_per_thread_t * rt,
 572                          ip4_full_reass_t * reass, u32 * bi0,
 573                          u32 * next0, u32 * error0, bool is_custom)
 574 {
 575   vlib_buffer_t *first_b = vlib_get_buffer (vm, reass->first_bi);
 576   vlib_buffer_t *last_b = NULL;
 577   u32 sub_chain_bi = reass->first_bi;
 578   u32 total_length = 0;
 579   u32 buf_cnt = 0;
 580   do
 581     {
 582       u32 tmp_bi = sub_chain_bi;
 583       vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
 584       ip4_header_t *ip = vlib_buffer_get_current (tmp);
 585       vnet_buffer_opaque_t *vnb = vnet_buffer (tmp);
 586       if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
 587           !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
 588         {
 589           return IP4_REASS_RC_INTERNAL_ERROR;
 590         }
 591
 592       u32 data_len = ip4_full_reass_buffer_get_data_len (tmp);
 593       u32 trim_front =
 594         ip4_header_bytes (ip) + ip4_full_reass_buffer_get_data_offset (tmp);
 595       u32 trim_end =
 596         vlib_buffer_length_in_chain (vm, tmp) - trim_front - data_len;
 597       if (tmp_bi == reass->first_bi)
 598         {
 599           /* first buffer - keep ip4 header */
 600           if (0 != ip4_full_reass_buffer_get_data_offset (tmp))
 601             {
 602               return IP4_REASS_RC_INTERNAL_ERROR;
 603             }
 604           trim_front = 0;
 605           trim_end = vlib_buffer_length_in_chain (vm, tmp) - data_len -
 606             ip4_header_bytes (ip);
 607           if (!(vlib_buffer_length_in_chain (vm, tmp) - trim_end > 0))
 608             {
 609               return IP4_REASS_RC_INTERNAL_ERROR;
 610             }
 611         }
 612       u32 keep_data =
 613         vlib_buffer_length_in_chain (vm, tmp) - trim_front - trim_end;
 614       while (1)
 615         {
 616           ++buf_cnt;
 617           if (trim_front)
 618             {
 619               if (trim_front > tmp->current_length)
 620                 {
 621                   /* drop whole buffer */
 622                   u32 to_be_freed_bi = tmp_bi;
 623                   trim_front -= tmp->current_length;
 624                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 625                     {
 626                       return IP4_REASS_RC_INTERNAL_ERROR;
 627                     }
 628                   tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 629                   tmp_bi = tmp->next_buffer;
 630                   tmp->next_buffer = 0;
 631                   tmp = vlib_get_buffer (vm, tmp_bi);
 632                   vlib_buffer_free_one (vm, to_be_freed_bi);
 633                   continue;
 634                 }
 635               else
 636                 {
 637                   vlib_buffer_advance (tmp, trim_front);
 638                   trim_front = 0;
 639                 }
 640             }
 641           if (keep_data)
 642             {
 643               if (last_b)
 644                 {
 645                   last_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
 646                   last_b->next_buffer = tmp_bi;
 647                 }
 648               last_b = tmp;
 649               if (keep_data <= tmp->current_length)
 650                 {
 651                   tmp->current_length = keep_data;
 652                   keep_data = 0;
 653                 }
 654               else
 655                 {
 656                   keep_data -= tmp->current_length;
 657                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 658                     {
 659                       return IP4_REASS_RC_INTERNAL_ERROR;
 660                     }
 661                 }
 662               total_length += tmp->current_length;
 663               if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
 664                 {
 665                   tmp_bi = tmp->next_buffer;
 666                   tmp = vlib_get_buffer (vm, tmp->next_buffer);
 667                 }
 668               else
 669                 {
 670                   break;
 671                 }
 672             }
 673           else
 674             {
 675               u32 to_be_freed_bi = tmp_bi;
 676               if (reass->first_bi == tmp_bi)
 677                 {
 678                   return IP4_REASS_RC_INTERNAL_ERROR;
 679                 }
 680               if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
 681                 {
 682                   tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 683                   tmp_bi = tmp->next_buffer;
 684                   tmp->next_buffer = 0;
 685                   tmp = vlib_get_buffer (vm, tmp_bi);
 686                   vlib_buffer_free_one (vm, to_be_freed_bi);
 687                 }
 688               else
 689                 {
 690                   tmp->next_buffer = 0;
 691                   vlib_buffer_free_one (vm, to_be_freed_bi);
 692                   break;
 693                 }
 694             }
 695         }
 696       sub_chain_bi =
 697         vnet_buffer (vlib_get_buffer (vm, sub_chain_bi))->ip.
 698         reass.next_range_bi;
 699     }
 700   while (~0 != sub_chain_bi);
 701
 702   if (!last_b)
 703     {
 704       return IP4_REASS_RC_INTERNAL_ERROR;
 705     }
 706   last_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 707
 708   if (total_length < first_b->current_length)
 709     {
 710       return IP4_REASS_RC_INTERNAL_ERROR;
 711     }
 712   total_length -= first_b->current_length;
 713   first_b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 714   first_b->total_length_not_including_first_buffer = total_length;
 715   ip4_header_t *ip = vlib_buffer_get_current (first_b);
 716   ip->flags_and_fragment_offset = 0;
 717   ip->length = clib_host_to_net_u16 (first_b->current_length + total_length);
 718   ip->checksum = ip4_header_checksum (ip);
 719   if (!vlib_buffer_chain_linearize (vm, first_b))
 720     {
 721       return IP4_REASS_RC_NO_BUF;
 722     }
 723   // reset to reconstruct the mbuf linking
 724   first_b->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
 725   if (PREDICT_FALSE (first_b->flags & VLIB_BUFFER_IS_TRACED))
 726     {
 727       ip4_full_reass_add_trace (vm, node, reass, reass->first_bi, FINALIZE, 0,
 728                                 ~0);
 729 #if 0
 730       // following code does a hexdump of packet fragments to stdout ...
 731       do
 732         {
 733           u32 bi = reass->first_bi;
 734           u8 *s = NULL;
 735           while (~0 != bi)
 736             {
 737               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 738               s = format (s, "%u: %U\n", bi, format_hexdump,
 739                           vlib_buffer_get_current (b), b->current_length);
 740               if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 741                 {
 742                   bi = b->next_buffer;
 743                 }
 744               else
 745                 {
 746                   break;
 747                 }
 748             }
 749           printf ("%.*s\n", vec_len (s), s);
 750           fflush (stdout);
 751           vec_free (s);
 752         }
 753       while (0);
 754 #endif
 755     }
 756   *bi0 = reass->first_bi;
 757   if (!is_custom)
 758     {
 759       *next0 = IP4_FULL_REASS_NEXT_INPUT;
 760     }
 761   else
 762     {
 763       *next0 = reass->next_index;
 764     }
 765   vnet_buffer (first_b)->ip.reass.estimated_mtu = reass->min_fragment_length;
 766   *error0 = IP4_ERROR_NONE;
 767   ip4_full_reass_free (rm, rt, reass);
 768   reass = NULL;
 769   return IP4_REASS_RC_OK;
 770 }
 771
 772 always_inline ip4_full_reass_rc_t
 773 ip4_full_reass_insert_range_in_chain (vlib_main_t * vm,
 774                                       ip4_full_reass_t * reass,
 775                                       u32 prev_range_bi, u32 new_next_bi)
 776 {
 777   vlib_buffer_t *new_next_b = vlib_get_buffer (vm, new_next_bi);
 778   vnet_buffer_opaque_t *new_next_vnb = vnet_buffer (new_next_b);
 779   if (~0 != prev_range_bi)
 780     {
 781       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 782       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 783       new_next_vnb->ip.reass.next_range_bi = prev_vnb->ip.reass.next_range_bi;
 784       prev_vnb->ip.reass.next_range_bi = new_next_bi;
 785     }
 786   else
 787     {
 788       if (~0 != reass->first_bi)
 789         {
 790           new_next_vnb->ip.reass.next_range_bi = reass->first_bi;
 791         }
 792       reass->first_bi = new_next_bi;
 793     }
 794   vnet_buffer_opaque_t *vnb = vnet_buffer (new_next_b);
 795   if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
 796       !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
 797     {
 798       return IP4_REASS_RC_INTERNAL_ERROR;
 799     }
 800   reass->data_len += ip4_full_reass_buffer_get_data_len (new_next_b);
 801   return IP4_REASS_RC_OK;
 802 }
 803
 804 always_inline ip4_full_reass_rc_t
 805 ip4_full_reass_remove_range_from_chain (vlib_main_t * vm,
 806                                         vlib_node_runtime_t * node,
 807                                         ip4_full_reass_t * reass,
 808                                         u32 prev_range_bi, u32 discard_bi)
 809 {
 810   vlib_buffer_t *discard_b = vlib_get_buffer (vm, discard_bi);
 811   vnet_buffer_opaque_t *discard_vnb = vnet_buffer (discard_b);
 812   if (~0 != prev_range_bi)
 813     {
 814       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 815       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 816       if (!(prev_vnb->ip.reass.next_range_bi == discard_bi))
 817         {
 818           return IP4_REASS_RC_INTERNAL_ERROR;
 819         }
 820       prev_vnb->ip.reass.next_range_bi = discard_vnb->ip.reass.next_range_bi;
 821     }
 822   else
 823     {
 824       reass->first_bi = discard_vnb->ip.reass.next_range_bi;
 825     }
 826   vnet_buffer_opaque_t *vnb = vnet_buffer (discard_b);
 827   if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
 828       !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
 829     {
 830       return IP4_REASS_RC_INTERNAL_ERROR;
 831     }
 832   reass->data_len -= ip4_full_reass_buffer_get_data_len (discard_b);
 833   while (1)
 834     {
 835       u32 to_be_freed_bi = discard_bi;
 836       if (PREDICT_FALSE (discard_b->flags & VLIB_BUFFER_IS_TRACED))
 837         {
 838           ip4_full_reass_add_trace (vm, node, reass, discard_bi, RANGE_DISCARD,
 839                                     0, ~0);
 840         }
 841       if (discard_b->flags & VLIB_BUFFER_NEXT_PRESENT)
 842         {
 843           discard_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 844           discard_bi = discard_b->next_buffer;
 845           discard_b->next_buffer = 0;
 846           discard_b = vlib_get_buffer (vm, discard_bi);
 847           vlib_buffer_free_one (vm, to_be_freed_bi);
 848         }
 849       else
 850         {
 851           discard_b->next_buffer = 0;
 852           vlib_buffer_free_one (vm, to_be_freed_bi);
 853           break;
 854         }
 855     }
 856   return IP4_REASS_RC_OK;
 857 }
 858
 859 always_inline ip4_full_reass_rc_t
 860 ip4_full_reass_update (vlib_main_t * vm, vlib_node_runtime_t * node,
 861                        ip4_full_reass_main_t * rm,
 862                        ip4_full_reass_per_thread_t * rt,
 863                        ip4_full_reass_t * reass, u32 * bi0, u32 * next0,
 864                        u32 * error0, bool is_custom, u32 * handoff_thread_idx)
 865 {
 866   vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
 867   vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
 868   if (is_custom)
 869     {
 870       // store (error_)next_index before it's overwritten
 871       reass->next_index = fvnb->ip.reass.next_index;
 872       reass->error_next_index = fvnb->ip.reass.error_next_index;
 873     }
 874   ip4_full_reass_rc_t rc = IP4_REASS_RC_OK;
 875   int consumed = 0;
 876   ip4_header_t *fip = vlib_buffer_get_current (fb);
 877   const u32 fragment_first = ip4_get_fragment_offset_bytes (fip);
 878   const u32 fragment_length =
 879     clib_net_to_host_u16 (fip->length) - ip4_header_bytes (fip);
 880   const u32 fragment_last = fragment_first + fragment_length - 1;
 881   fvnb->ip.reass.fragment_first = fragment_first;
 882   fvnb->ip.reass.fragment_last = fragment_last;
 883   int more_fragments = ip4_get_fragment_more (fip);
 884   u32 candidate_range_bi = reass->first_bi;
 885   u32 prev_range_bi = ~0;
 886   fvnb->ip.reass.range_first = fragment_first;
 887   fvnb->ip.reass.range_last = fragment_last;
 888   fvnb->ip.reass.next_range_bi = ~0;
 889   if (!more_fragments)
 890     {
 891       reass->last_packet_octet = fragment_last;
 892     }
 893   if (~0 == reass->first_bi)
 894     {
 895       // starting a new reassembly
 896       rc =
 897         ip4_full_reass_insert_range_in_chain (vm, reass, prev_range_bi, *bi0);
 898       if (IP4_REASS_RC_OK != rc)
 899         {
 900           return rc;
 901         }
 902       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 903         {
 904           ip4_full_reass_add_trace (vm, node, reass, *bi0, RANGE_NEW, 0, ~0);
 905         }
 906       *bi0 = ~0;
 907       reass->min_fragment_length = clib_net_to_host_u16 (fip->length);
 908       reass->fragments_n = 1;
 909       return IP4_REASS_RC_OK;
 910     }
 911   reass->min_fragment_length =
 912     clib_min (clib_net_to_host_u16 (fip->length),
 913               fvnb->ip.reass.estimated_mtu);
 914   while (~0 != candidate_range_bi)
 915     {
 916       vlib_buffer_t *candidate_b = vlib_get_buffer (vm, candidate_range_bi);
 917       vnet_buffer_opaque_t *candidate_vnb = vnet_buffer (candidate_b);
 918       if (fragment_first > candidate_vnb->ip.reass.range_last)
 919         {
 920           // this fragments starts after candidate range
 921           prev_range_bi = candidate_range_bi;
 922           candidate_range_bi = candidate_vnb->ip.reass.next_range_bi;
 923           if (candidate_vnb->ip.reass.range_last < fragment_last &&
 924               ~0 == candidate_range_bi)
 925             {
 926               // special case - this fragment falls beyond all known ranges
 927               rc = ip4_full_reass_insert_range_in_chain (vm, reass,
 928                                                          prev_range_bi, *bi0);
 929               if (IP4_REASS_RC_OK != rc)
 930                 {
 931                   return rc;
 932                 }
 933               consumed = 1;
 934               break;
 935             }
 936           continue;
 937         }
 938       if (fragment_last < candidate_vnb->ip.reass.range_first)
 939         {
 940           // this fragment ends before candidate range without any overlap
 941           rc = ip4_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
 942                                                      *bi0);
 943           if (IP4_REASS_RC_OK != rc)
 944             {
 945               return rc;
 946             }
 947           consumed = 1;
 948         }
 949       else
 950         {
 951           if (fragment_first >= candidate_vnb->ip.reass.range_first &&
 952               fragment_last <= candidate_vnb->ip.reass.range_last)
 953             {
 954               // this fragment is a (sub)part of existing range, ignore it
 955               if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 956                 {
 957                   ip4_full_reass_add_trace (vm, node, reass, *bi0,
 958                                             RANGE_OVERLAP, 0, ~0);
 959                 }
 960               break;
 961             }
 962           int discard_candidate = 0;
 963           if (fragment_first < candidate_vnb->ip.reass.range_first)
 964             {
 965               u32 overlap =
 966                 fragment_last - candidate_vnb->ip.reass.range_first + 1;
 967               if (overlap < ip4_full_reass_buffer_get_data_len (candidate_b))
 968                 {
 969                   candidate_vnb->ip.reass.range_first += overlap;
 970                   if (reass->data_len < overlap)
 971                     {
 972                       return IP4_REASS_RC_INTERNAL_ERROR;
 973                     }
 974                   reass->data_len -= overlap;
 975                   if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 976                     {
 977                       ip4_full_reass_add_trace (vm, node, reass,
 978                                                 candidate_range_bi,
 979                                                 RANGE_SHRINK, 0, ~0);
 980                     }
 981                   rc = ip4_full_reass_insert_range_in_chain (
 982                     vm, reass, prev_range_bi, *bi0);
 983                   if (IP4_REASS_RC_OK != rc)
 984                     {
 985                       return rc;
 986                     }
 987                   consumed = 1;
 988                 }
 989               else
 990                 {
 991                   discard_candidate = 1;
 992                 }
 993             }
 994           else if (fragment_last > candidate_vnb->ip.reass.range_last)
 995             {
 996               u32 overlap =
 997                 candidate_vnb->ip.reass.range_last - fragment_first + 1;
 998               if (overlap < ip4_full_reass_buffer_get_data_len (candidate_b))
 999                 {
1000                   fvnb->ip.reass.range_first += overlap;
1001                   if (~0 != candidate_vnb->ip.reass.next_range_bi)
1002                     {
1003                       prev_range_bi = candidate_range_bi;
1004                       candidate_range_bi =
1005                         candidate_vnb->ip.reass.next_range_bi;
1006                       continue;
1007                     }
1008                   else
1009                     {
1010                       // special case - last range discarded
1011                       rc = ip4_full_reass_insert_range_in_chain (
1012                         vm, reass, candidate_range_bi, *bi0);
1013                       if (IP4_REASS_RC_OK != rc)
1014                         {
1015                           return rc;
1016                         }
1017                       consumed = 1;
1018                     }
1019                 }
1020               else
1021                 {
1022                   discard_candidate = 1;
1023                 }
1024             }
1025           else
1026             {
1027               discard_candidate = 1;
1028             }
1029           if (discard_candidate)
1030             {
1031               u32 next_range_bi = candidate_vnb->ip.reass.next_range_bi;
1032               // discard candidate range, probe next range
1033               rc = ip4_full_reass_remove_range_from_chain (
1034                 vm, node, reass, prev_range_bi, candidate_range_bi);
1035               if (IP4_REASS_RC_OK != rc)
1036                 {
1037                   return rc;
1038                 }
1039               if (~0 != next_range_bi)
1040                 {
1041                   candidate_range_bi = next_range_bi;
1042                   continue;
1043                 }
1044               else
1045                 {
1046                   // special case - last range discarded
1047                   rc = ip4_full_reass_insert_range_in_chain (
1048                     vm, reass, prev_range_bi, *bi0);
1049                   if (IP4_REASS_RC_OK != rc)
1050                     {
1051                       return rc;
1052                     }
1053                   consumed = 1;
1054                 }
1055             }
1056         }
1057       break;
1058     }
1059   ++reass->fragments_n;
1060   if (consumed)
1061     {
1062       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
1063         {
1064           ip4_full_reass_add_trace (vm, node, reass, *bi0, RANGE_NEW, 0, ~0);
1065         }
1066     }
1067   if (~0 != reass->last_packet_octet &&
1068       reass->data_len == reass->last_packet_octet + 1)
1069     {
1070       *handoff_thread_idx = reass->sendout_thread_index;
1071       int handoff =
1072         reass->memory_owner_thread_index != reass->sendout_thread_index;
1073       rc =
1074         ip4_full_reass_finalize (vm, node, rm, rt, reass, bi0, next0, error0,
1075                                  is_custom);
1076       if (IP4_REASS_RC_OK == rc && handoff)
1077         {
1078           rc = IP4_REASS_RC_HANDOFF;
1079         }
1080     }
1081   else
1082     {
1083       if (consumed)
1084         {
1085           *bi0 = ~0;
1086           if (reass->fragments_n > rm->max_reass_len)
1087             {
1088               rc = IP4_REASS_RC_TOO_MANY_FRAGMENTS;
1089             }
1090         }
1091       else
1092         {
1093           *next0 = IP4_FULL_REASS_NEXT_DROP;
1094           *error0 = IP4_ERROR_REASS_DUPLICATE_FRAGMENT;
1095         }
1096     }
1097   return rc;
1098 }
1099
1100 always_inline uword
1101 ip4_full_reass_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1102                        vlib_frame_t *frame, ip4_full_reass_node_type_t type,
1103                        bool is_local)
1104 {
1105   u32 *from = vlib_frame_vector_args (frame);
1106   u32 n_left_from, n_left_to_next, *to_next, next_index;
1107   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1108   ip4_full_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
1109   clib_spinlock_lock (&rt->lock);
1110
1111   n_left_from = frame->n_vectors;
1112   next_index = node->cached_next_index;
1113   while (n_left_from > 0)
1114     {
1115       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1116
1117       while (n_left_from > 0 && n_left_to_next > 0)
1118         {
1119           u32 bi0;
1120           vlib_buffer_t *b0;
1121           u32 next0;
1122           u32 error0 = IP4_ERROR_NONE;
1123
1124           bi0 = from[0];
1125           b0 = vlib_get_buffer (vm, bi0);
1126
1127           ip4_header_t *ip0 = vlib_buffer_get_current (b0);
1128           if (!ip4_get_fragment_more (ip0) && !ip4_get_fragment_offset (ip0))
1129             {
1130               // this is a whole packet - no fragmentation
1131               if (CUSTOM != type)
1132                 {
1133                   next0 = IP4_FULL_REASS_NEXT_INPUT;
1134                 }
1135               else
1136                 {
1137                   next0 = vnet_buffer (b0)->ip.reass.next_index;
1138                 }
1139               ip4_full_reass_add_trace (vm, node, NULL, bi0, PASSTHROUGH, 0,
1140                                         ~0);
1141               goto packet_enqueue;
1142             }
1143
1144           if (is_local && !rm->is_local_reass_enabled)
1145             {
1146               next0 = IP4_FULL_REASS_NEXT_DROP;
1147               goto packet_enqueue;
1148             }
1149
1150           const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
1151           const u32 fragment_length =
1152             clib_net_to_host_u16 (ip0->length) - ip4_header_bytes (ip0);
1153           const u32 fragment_last = fragment_first + fragment_length - 1;
1154           if (fragment_first > fragment_last || fragment_first + fragment_length > UINT16_MAX - 20 || (fragment_length < 8 && ip4_get_fragment_more (ip0)))     // 8 is minimum frag length per RFC 791
1155             {
1156               next0 = IP4_FULL_REASS_NEXT_DROP;
1157               error0 = IP4_ERROR_REASS_MALFORMED_PACKET;
1158               goto packet_enqueue;
1159             }
1160           ip4_full_reass_kv_t kv;
1161           u8 do_handoff = 0;
1162
1163           kv.k.as_u64[0] =
1164             (u64) vec_elt (ip4_main.fib_index_by_sw_if_index,
1165                            vnet_buffer (b0)->sw_if_index[VLIB_RX]) |
1166             (u64) ip0->src_address.as_u32 << 32;
1167           kv.k.as_u64[1] =
1168             (u64) ip0->dst_address.
1169             as_u32 | (u64) ip0->fragment_id << 32 | (u64) ip0->protocol << 48;
1170
1171           ip4_full_reass_t *reass =
1172             ip4_full_reass_find_or_create (vm, node, rm, rt, &kv,
1173                                            &do_handoff);
1174
1175           if (reass)
1176             {
1177               const u32 fragment_first = ip4_get_fragment_offset_bytes (ip0);
1178               if (0 == fragment_first)
1179                 {
1180                   reass->sendout_thread_index = vm->thread_index;
1181                 }
1182             }
1183
1184           if (PREDICT_FALSE (do_handoff))
1185             {
1186               next0 = IP4_FULL_REASS_NEXT_HANDOFF;
1187               vnet_buffer (b0)->ip.reass.owner_thread_index =
1188                 kv.v.memory_owner_thread_index;
1189             }
1190           else if (reass)
1191             {
1192               u32 handoff_thread_idx;
1193               u32 counter = ~0;
1194               switch (ip4_full_reass_update
1195                       (vm, node, rm, rt, reass, &bi0, &next0,
1196                        &error0, CUSTOM == type, &handoff_thread_idx))
1197                 {
1198                 case IP4_REASS_RC_OK:
1199                   /* nothing to do here */
1200                   break;
1201                 case IP4_REASS_RC_HANDOFF:
1202                   next0 = IP4_FULL_REASS_NEXT_HANDOFF;
1203                   b0 = vlib_get_buffer (vm, bi0);
1204                   vnet_buffer (b0)->ip.reass.owner_thread_index =
1205                     handoff_thread_idx;
1206                   break;
1207                 case IP4_REASS_RC_TOO_MANY_FRAGMENTS:
1208                   counter = IP4_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG;
1209                   break;
1210                 case IP4_REASS_RC_NO_BUF:
1211                   counter = IP4_ERROR_REASS_NO_BUF;
1212                   break;
1213                 case IP4_REASS_RC_INTERNAL_ERROR:
1214                   counter = IP4_ERROR_REASS_INTERNAL_ERROR;
1215                   break;
1216                 }
1217
1218               if (~0 != counter)
1219                 {
1220                   vlib_node_increment_counter (vm, node->node_index, counter,
1221                                                1);
1222                   ip4_full_reass_drop_all (vm, node, reass);
1223                   ip4_full_reass_free (rm, rt, reass);
1224                   goto next_packet;
1225                 }
1226             }
1227           else
1228             {
1229               next0 = IP4_FULL_REASS_NEXT_DROP;
1230               error0 = IP4_ERROR_REASS_LIMIT_REACHED;
1231             }
1232
1233
1234         packet_enqueue:
1235
1236           if (bi0 != ~0)
1237             {
1238               to_next[0] = bi0;
1239               to_next += 1;
1240               n_left_to_next -= 1;
1241
1242               /* bi0 might have been updated by reass_finalize, reload */
1243               b0 = vlib_get_buffer (vm, bi0);
1244               if (IP4_ERROR_NONE != error0)
1245                 {
1246                   b0->error = node->errors[error0];
1247                 }
1248
1249               if (next0 == IP4_FULL_REASS_NEXT_HANDOFF)
1250                 {
1251                   if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1252                     {
1253                       ip4_full_reass_add_trace (
1254                         vm, node, NULL, bi0, HANDOFF, 0,
1255                         vnet_buffer (b0)->ip.reass.owner_thread_index);
1256                     }
1257                 }
1258               else if (FEATURE == type && IP4_ERROR_NONE == error0)
1259                 {
1260                   vnet_feature_next (&next0, b0);
1261                 }
1262               vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1263                                                to_next, n_left_to_next,
1264                                                bi0, next0);
1265               IP4_REASS_DEBUG_BUFFER (bi0, enqueue_next);
1266             }
1267
1268         next_packet:
1269           from += 1;
1270           n_left_from -= 1;
1271         }
1272
1273       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1274     }
1275
1276   clib_spinlock_unlock (&rt->lock);
1277   return frame->n_vectors;
1278 }
1279
1280 static char *ip4_full_reass_error_strings[] = {
1281 #define _(sym, string) string,
1282   foreach_ip4_error
1283 #undef _
1284 };
1285
1286 VLIB_NODE_FN (ip4_full_reass_node) (vlib_main_t * vm,
1287                                     vlib_node_runtime_t * node,
1288                                     vlib_frame_t * frame)
1289 {
1290   return ip4_full_reass_inline (vm, node, frame, NORMAL, false /* is_local */);
1291 }
1292
1293 VLIB_REGISTER_NODE (ip4_full_reass_node) = {
1294     .name = "ip4-full-reassembly",
1295     .vector_size = sizeof (u32),
1296     .format_trace = format_ip4_full_reass_trace,
1297     .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1298     .error_strings = ip4_full_reass_error_strings,
1299     .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1300     .next_nodes =
1301         {
1302                 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1303                 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1304                 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reassembly-handoff",
1305
1306         },
1307 };
1308
1309 VLIB_NODE_FN (ip4_local_full_reass_node)
1310 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1311 {
1312   return ip4_full_reass_inline (vm, node, frame, NORMAL, true /* is_local */);
1313 }
1314
1315 VLIB_REGISTER_NODE (ip4_local_full_reass_node) = {
1316     .name = "ip4-local-full-reassembly",
1317     .vector_size = sizeof (u32),
1318     .format_trace = format_ip4_full_reass_trace,
1319     .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1320     .error_strings = ip4_full_reass_error_strings,
1321     .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1322     .next_nodes =
1323         {
1324                 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1325                 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1326                 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-local-full-reassembly-handoff",
1327
1328         },
1329 };
1330
1331 VLIB_NODE_FN (ip4_full_reass_node_feature) (vlib_main_t * vm,
1332                                             vlib_node_runtime_t * node,
1333                                             vlib_frame_t * frame)
1334 {
1335   return ip4_full_reass_inline (vm, node, frame, FEATURE,
1336                                 false /* is_local */);
1337 }
1338
1339 VLIB_REGISTER_NODE (ip4_full_reass_node_feature) = {
1340     .name = "ip4-full-reassembly-feature",
1341     .vector_size = sizeof (u32),
1342     .format_trace = format_ip4_full_reass_trace,
1343     .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1344     .error_strings = ip4_full_reass_error_strings,
1345     .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1346     .next_nodes =
1347         {
1348                 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1349                 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1350                 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reass-feature-hoff",
1351         },
1352 };
1353
1354 VNET_FEATURE_INIT (ip4_full_reass_feature, static) = {
1355     .arc_name = "ip4-unicast",
1356     .node_name = "ip4-full-reassembly-feature",
1357     .runs_before = VNET_FEATURES ("ip4-lookup",
1358                                   "ipsec4-input-feature"),
1359     .runs_after = 0,
1360 };
1361
1362 VLIB_NODE_FN (ip4_full_reass_node_custom) (vlib_main_t * vm,
1363                                            vlib_node_runtime_t * node,
1364                                            vlib_frame_t * frame)
1365 {
1366   return ip4_full_reass_inline (vm, node, frame, CUSTOM, false /* is_local */);
1367 }
1368
1369 VLIB_REGISTER_NODE (ip4_full_reass_node_custom) = {
1370     .name = "ip4-full-reassembly-custom",
1371     .vector_size = sizeof (u32),
1372     .format_trace = format_ip4_full_reass_trace,
1373     .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1374     .error_strings = ip4_full_reass_error_strings,
1375     .n_next_nodes = IP4_FULL_REASS_N_NEXT,
1376     .next_nodes =
1377         {
1378                 [IP4_FULL_REASS_NEXT_INPUT] = "ip4-input",
1379                 [IP4_FULL_REASS_NEXT_DROP] = "ip4-drop",
1380                 [IP4_FULL_REASS_NEXT_HANDOFF] = "ip4-full-reass-custom-hoff",
1381         },
1382 };
1383
1384 VNET_FEATURE_INIT (ip4_full_reass_custom, static) = {
1385     .arc_name = "ip4-unicast",
1386     .node_name = "ip4-full-reassembly-feature",
1387     .runs_before = VNET_FEATURES ("ip4-lookup",
1388                                   "ipsec4-input-feature"),
1389     .runs_after = 0,
1390 };
1391
1392
1393 #ifndef CLIB_MARCH_VARIANT
1394 uword
1395 ip4_full_reass_custom_register_next_node (uword node_index)
1396 {
1397   return vlib_node_add_next (vlib_get_main (),
1398                              ip4_full_reass_node_custom.index, node_index);
1399 }
1400
1401 always_inline u32
1402 ip4_full_reass_get_nbuckets ()
1403 {
1404   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1405   u32 nbuckets;
1406   u8 i;
1407
1408   nbuckets = (u32) (rm->max_reass_n / IP4_REASS_HT_LOAD_FACTOR);
1409
1410   for (i = 0; i < 31; i++)
1411     if ((1 << i) >= nbuckets)
1412       break;
1413   nbuckets = 1 << i;
1414
1415   return nbuckets;
1416 }
1417 #endif /* CLIB_MARCH_VARIANT */
1418
1419 typedef enum
1420 {
1421   IP4_EVENT_CONFIG_CHANGED = 1,
1422 } ip4_full_reass_event_t;
1423
1424 typedef struct
1425 {
1426   int failure;
1427   clib_bihash_16_8_t *new_hash;
1428 } ip4_rehash_cb_ctx;
1429
1430 #ifndef CLIB_MARCH_VARIANT
1431 static int
1432 ip4_rehash_cb (clib_bihash_kv_16_8_t * kv, void *_ctx)
1433 {
1434   ip4_rehash_cb_ctx *ctx = _ctx;
1435   if (clib_bihash_add_del_16_8 (ctx->new_hash, kv, 1))
1436     {
1437       ctx->failure = 1;
1438     }
1439   return (BIHASH_WALK_CONTINUE);
1440 }
1441
1442 static void
1443 ip4_full_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
1444                            u32 max_reassembly_length,
1445                            u32 expire_walk_interval_ms)
1446 {
1447   ip4_full_reass_main.timeout_ms = timeout_ms;
1448   ip4_full_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
1449   ip4_full_reass_main.max_reass_n = max_reassemblies;
1450   ip4_full_reass_main.max_reass_len = max_reassembly_length;
1451   ip4_full_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
1452 }
1453
1454 vnet_api_error_t
1455 ip4_full_reass_set (u32 timeout_ms, u32 max_reassemblies,
1456                     u32 max_reassembly_length, u32 expire_walk_interval_ms)
1457 {
1458   u32 old_nbuckets = ip4_full_reass_get_nbuckets ();
1459   ip4_full_reass_set_params (timeout_ms, max_reassemblies,
1460                              max_reassembly_length, expire_walk_interval_ms);
1461   vlib_process_signal_event (ip4_full_reass_main.vlib_main,
1462                              ip4_full_reass_main.ip4_full_reass_expire_node_idx,
1463                              IP4_EVENT_CONFIG_CHANGED, 0);
1464   u32 new_nbuckets = ip4_full_reass_get_nbuckets ();
1465   if (ip4_full_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
1466     {
1467       clib_bihash_16_8_t new_hash;
1468       clib_memset (&new_hash, 0, sizeof (new_hash));
1469       ip4_rehash_cb_ctx ctx;
1470       ctx.failure = 0;
1471       ctx.new_hash = &new_hash;
1472       clib_bihash_init_16_8 (&new_hash, "ip4-dr", new_nbuckets,
1473                              new_nbuckets * 1024);
1474       clib_bihash_foreach_key_value_pair_16_8 (&ip4_full_reass_main.hash,
1475                                                ip4_rehash_cb, &ctx);
1476       if (ctx.failure)
1477         {
1478           clib_bihash_free_16_8 (&new_hash);
1479           return -1;
1480         }
1481       else
1482         {
1483           clib_bihash_free_16_8 (&ip4_full_reass_main.hash);
1484           clib_memcpy_fast (&ip4_full_reass_main.hash, &new_hash,
1485                             sizeof (ip4_full_reass_main.hash));
1486           clib_bihash_copied (&ip4_full_reass_main.hash, &new_hash);
1487         }
1488     }
1489   return 0;
1490 }
1491
1492 vnet_api_error_t
1493 ip4_full_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
1494                     u32 * max_reassembly_length,
1495                     u32 * expire_walk_interval_ms)
1496 {
1497   *timeout_ms = ip4_full_reass_main.timeout_ms;
1498   *max_reassemblies = ip4_full_reass_main.max_reass_n;
1499   *max_reassembly_length = ip4_full_reass_main.max_reass_len;
1500   *expire_walk_interval_ms = ip4_full_reass_main.expire_walk_interval_ms;
1501   return 0;
1502 }
1503
1504 static clib_error_t *
1505 ip4_full_reass_init_function (vlib_main_t * vm)
1506 {
1507   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1508   clib_error_t *error = 0;
1509   u32 nbuckets;
1510   vlib_node_t *node;
1511
1512   rm->vlib_main = vm;
1513
1514   vec_validate (rm->per_thread_data, vlib_num_workers ());
1515   ip4_full_reass_per_thread_t *rt;
1516   vec_foreach (rt, rm->per_thread_data)
1517   {
1518     clib_spinlock_init (&rt->lock);
1519     pool_alloc (rt->pool, rm->max_reass_n);
1520   }
1521
1522   node = vlib_get_node_by_name (vm, (u8 *) "ip4-full-reassembly-expire-walk");
1523   ASSERT (node);
1524   rm->ip4_full_reass_expire_node_idx = node->index;
1525
1526   ip4_full_reass_set_params (IP4_REASS_TIMEOUT_DEFAULT_MS,
1527                              IP4_REASS_MAX_REASSEMBLIES_DEFAULT,
1528                              IP4_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
1529                              IP4_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
1530
1531   nbuckets = ip4_full_reass_get_nbuckets ();
1532   clib_bihash_init_16_8 (&rm->hash, "ip4-dr", nbuckets, nbuckets * 1024);
1533
1534   node = vlib_get_node_by_name (vm, (u8 *) "ip4-drop");
1535   ASSERT (node);
1536   rm->ip4_drop_idx = node->index;
1537
1538   rm->fq_index = vlib_frame_queue_main_init (ip4_full_reass_node.index, 0);
1539   rm->fq_local_index =
1540     vlib_frame_queue_main_init (ip4_local_full_reass_node.index, 0);
1541   rm->fq_feature_index =
1542     vlib_frame_queue_main_init (ip4_full_reass_node_feature.index, 0);
1543   rm->fq_custom_index =
1544     vlib_frame_queue_main_init (ip4_full_reass_node_custom.index, 0);
1545
1546   rm->feature_use_refcount_per_intf = NULL;
1547   rm->is_local_reass_enabled = 1;
1548
1549   return error;
1550 }
1551
1552 VLIB_INIT_FUNCTION (ip4_full_reass_init_function);
1553 #endif /* CLIB_MARCH_VARIANT */
1554
1555 static uword
1556 ip4_full_reass_walk_expired (vlib_main_t *vm, vlib_node_runtime_t *node,
1557                              CLIB_UNUSED (vlib_frame_t *f))
1558 {
1559   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1560   uword event_type, *event_data = 0;
1561
1562   while (true)
1563     {
1564       vlib_process_wait_for_event_or_clock (vm,
1565                                             (f64)
1566                                             rm->expire_walk_interval_ms /
1567                                             (f64) MSEC_PER_SEC);
1568       event_type = vlib_process_get_events (vm, &event_data);
1569
1570       switch (event_type)
1571         {
1572         case ~0:
1573           /* no events => timeout */
1574           /* fallthrough */
1575         case IP4_EVENT_CONFIG_CHANGED:
1576           /* nothing to do here */
1577           break;
1578         default:
1579           clib_warning ("BUG: event type 0x%wx", event_type);
1580           break;
1581         }
1582       f64 now = vlib_time_now (vm);
1583
1584       ip4_full_reass_t *reass;
1585       int *pool_indexes_to_free = NULL;
1586
1587       uword thread_index = 0;
1588       int index;
1589       const uword nthreads = vlib_num_workers () + 1;
1590       for (thread_index = 0; thread_index < nthreads; ++thread_index)
1591         {
1592           ip4_full_reass_per_thread_t *rt =
1593             &rm->per_thread_data[thread_index];
1594           clib_spinlock_lock (&rt->lock);
1595
1596           vec_reset_length (pool_indexes_to_free);
1597           pool_foreach_index (index, rt->pool)  {
1598                                 reass = pool_elt_at_index (rt->pool, index);
1599                                 if (now > reass->last_heard + rm->timeout)
1600                                   {
1601                                     vec_add1 (pool_indexes_to_free, index);
1602                                   }
1603                               }
1604           int *i;
1605           vec_foreach (i, pool_indexes_to_free)
1606           {
1607             ip4_full_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1608             ip4_full_reass_drop_all (vm, node, reass);
1609             ip4_full_reass_free (rm, rt, reass);
1610           }
1611
1612           clib_spinlock_unlock (&rt->lock);
1613         }
1614
1615       vec_free (pool_indexes_to_free);
1616       if (event_data)
1617         {
1618           _vec_len (event_data) = 0;
1619         }
1620     }
1621
1622   return 0;
1623 }
1624
1625 VLIB_REGISTER_NODE (ip4_full_reass_expire_node) = {
1626     .function = ip4_full_reass_walk_expired,
1627     .type = VLIB_NODE_TYPE_PROCESS,
1628     .name = "ip4-full-reassembly-expire-walk",
1629     .format_trace = format_ip4_full_reass_trace,
1630     .n_errors = ARRAY_LEN (ip4_full_reass_error_strings),
1631     .error_strings = ip4_full_reass_error_strings,
1632
1633 };
1634
1635 static u8 *
1636 format_ip4_full_reass_key (u8 * s, va_list * args)
1637 {
1638   ip4_full_reass_key_t *key = va_arg (*args, ip4_full_reass_key_t *);
1639   s =
1640     format (s,
1641             "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1642             key->xx_id, format_ip4_address, &key->src, format_ip4_address,
1643             &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1644   return s;
1645 }
1646
1647 static u8 *
1648 format_ip4_reass (u8 * s, va_list * args)
1649 {
1650   vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1651   ip4_full_reass_t *reass = va_arg (*args, ip4_full_reass_t *);
1652
1653   s = format (s, "ID: %lu, key: %U\n  first_bi: %u, data_len: %u, "
1654               "last_packet_octet: %u, trace_op_counter: %u\n",
1655               reass->id, format_ip4_full_reass_key, &reass->key,
1656               reass->first_bi, reass->data_len,
1657               reass->last_packet_octet, reass->trace_op_counter);
1658
1659   u32 bi = reass->first_bi;
1660   u32 counter = 0;
1661   while (~0 != bi)
1662     {
1663       vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1664       vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1665       s =
1666         format (s,
1667                 "  #%03u: range: [%u, %u], bi: %u, off: %d, len: %u, "
1668                 "fragment[%u, %u]\n", counter, vnb->ip.reass.range_first,
1669                 vnb->ip.reass.range_last, bi,
1670                 ip4_full_reass_buffer_get_data_offset (b),
1671                 ip4_full_reass_buffer_get_data_len (b),
1672                 vnb->ip.reass.fragment_first, vnb->ip.reass.fragment_last);
1673       if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1674         {
1675           bi = b->next_buffer;
1676         }
1677       else
1678         {
1679           bi = ~0;
1680         }
1681     }
1682   return s;
1683 }
1684
1685 static clib_error_t *
1686 show_ip4_reass (vlib_main_t * vm,
1687                 unformat_input_t * input,
1688                 CLIB_UNUSED (vlib_cli_command_t * lmd))
1689 {
1690   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1691
1692   vlib_cli_output (vm, "---------------------");
1693   vlib_cli_output (vm, "IP4 reassembly status");
1694   vlib_cli_output (vm, "---------------------");
1695   bool details = false;
1696   if (unformat (input, "details"))
1697     {
1698       details = true;
1699     }
1700
1701   u32 sum_reass_n = 0;
1702   ip4_full_reass_t *reass;
1703   uword thread_index;
1704   const uword nthreads = vlib_num_workers () + 1;
1705   for (thread_index = 0; thread_index < nthreads; ++thread_index)
1706     {
1707       ip4_full_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1708       clib_spinlock_lock (&rt->lock);
1709       if (details)
1710         {
1711           pool_foreach (reass, rt->pool) {
1712             vlib_cli_output (vm, "%U", format_ip4_reass, vm, reass);
1713           }
1714         }
1715       sum_reass_n += rt->reass_n;
1716       clib_spinlock_unlock (&rt->lock);
1717     }
1718   vlib_cli_output (vm, "---------------------");
1719   vlib_cli_output (vm, "Current full IP4 reassemblies count: %lu\n",
1720                    (long unsigned) sum_reass_n);
1721   vlib_cli_output (vm,
1722                    "Maximum configured concurrent full IP4 reassemblies per worker-thread: %lu\n",
1723                    (long unsigned) rm->max_reass_n);
1724   vlib_cli_output (vm,
1725                    "Maximum configured amount of fragments "
1726                    "per full IP4 reassembly: %lu\n",
1727                    (long unsigned) rm->max_reass_len);
1728   vlib_cli_output (vm,
1729                    "Maximum configured full IP4 reassembly timeout: %lums\n",
1730                    (long unsigned) rm->timeout_ms);
1731   vlib_cli_output (vm,
1732                    "Maximum configured full IP4 reassembly expire walk interval: %lums\n",
1733                    (long unsigned) rm->expire_walk_interval_ms);
1734   return 0;
1735 }
1736
1737 VLIB_CLI_COMMAND (show_ip4_full_reass_cmd, static) = {
1738     .path = "show ip4-full-reassembly",
1739     .short_help = "show ip4-full-reassembly [details]",
1740     .function = show_ip4_reass,
1741 };
1742
1743 #ifndef CLIB_MARCH_VARIANT
1744 vnet_api_error_t
1745 ip4_full_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1746 {
1747   return vnet_feature_enable_disable ("ip4-unicast",
1748                                       "ip4-full-reassembly-feature",
1749                                       sw_if_index, enable_disable, 0, 0);
1750 }
1751 #endif /* CLIB_MARCH_VARIANT */
1752
1753
1754 #define foreach_ip4_full_reass_handoff_error                       \
1755 _(CONGESTION_DROP, "congestion drop")
1756
1757
1758 typedef enum
1759 {
1760 #define _(sym,str) IP4_FULL_REASS_HANDOFF_ERROR_##sym,
1761   foreach_ip4_full_reass_handoff_error
1762 #undef _
1763     IP4_FULL_REASS_HANDOFF_N_ERROR,
1764 } ip4_full_reass_handoff_error_t;
1765
1766 static char *ip4_full_reass_handoff_error_strings[] = {
1767 #define _(sym,string) string,
1768   foreach_ip4_full_reass_handoff_error
1769 #undef _
1770 };
1771
1772 typedef struct
1773 {
1774   u32 next_worker_index;
1775 } ip4_full_reass_handoff_trace_t;
1776
1777 static u8 *
1778 format_ip4_full_reass_handoff_trace (u8 * s, va_list * args)
1779 {
1780   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1781   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1782   ip4_full_reass_handoff_trace_t *t =
1783     va_arg (*args, ip4_full_reass_handoff_trace_t *);
1784
1785   s =
1786     format (s, "ip4-full-reassembly-handoff: next-worker %d",
1787             t->next_worker_index);
1788
1789   return s;
1790 }
1791
1792 always_inline uword
1793 ip4_full_reass_handoff_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1794                                     vlib_frame_t *frame,
1795                                     ip4_full_reass_node_type_t type,
1796                                     bool is_local)
1797 {
1798   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1799
1800   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1801   u32 n_enq, n_left_from, *from;
1802   u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1803   u32 fq_index;
1804
1805   from = vlib_frame_vector_args (frame);
1806   n_left_from = frame->n_vectors;
1807   vlib_get_buffers (vm, from, bufs, n_left_from);
1808
1809   b = bufs;
1810   ti = thread_indices;
1811
1812   switch (type)
1813     {
1814     case NORMAL:
1815       if (is_local)
1816         {
1817           fq_index = rm->fq_local_index;
1818         }
1819       else
1820         {
1821           fq_index = rm->fq_index;
1822         }
1823       break;
1824     case FEATURE:
1825       fq_index = rm->fq_feature_index;
1826       break;
1827     case CUSTOM:
1828       fq_index = rm->fq_custom_index;
1829       break;
1830     default:
1831       clib_warning ("Unexpected `type' (%d)!", type);
1832     }
1833
1834   while (n_left_from > 0)
1835     {
1836       ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1837
1838       if (PREDICT_FALSE
1839           ((node->flags & VLIB_NODE_FLAG_TRACE)
1840            && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1841         {
1842           ip4_full_reass_handoff_trace_t *t =
1843             vlib_add_trace (vm, node, b[0], sizeof (*t));
1844           t->next_worker_index = ti[0];
1845         }
1846
1847       n_left_from -= 1;
1848       ti += 1;
1849       b += 1;
1850     }
1851   n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1852                                          thread_indices, frame->n_vectors, 1);
1853
1854   if (n_enq < frame->n_vectors)
1855     vlib_node_increment_counter (vm, node->node_index,
1856                                  IP4_FULL_REASS_HANDOFF_ERROR_CONGESTION_DROP,
1857                                  frame->n_vectors - n_enq);
1858   return frame->n_vectors;
1859 }
1860
1861 VLIB_NODE_FN (ip4_full_reass_handoff_node) (vlib_main_t * vm,
1862                                             vlib_node_runtime_t * node,
1863                                             vlib_frame_t * frame)
1864 {
1865   return ip4_full_reass_handoff_node_inline (vm, node, frame, NORMAL,
1866                                              false /* is_local */);
1867 }
1868
1869
1870 VLIB_REGISTER_NODE (ip4_full_reass_handoff_node) = {
1871   .name = "ip4-full-reassembly-handoff",
1872   .vector_size = sizeof (u32),
1873   .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
1874   .error_strings = ip4_full_reass_handoff_error_strings,
1875   .format_trace = format_ip4_full_reass_handoff_trace,
1876
1877   .n_next_nodes = 1,
1878
1879   .next_nodes = {
1880     [0] = "error-drop",
1881   },
1882 };
1883
1884 VLIB_NODE_FN (ip4_local_full_reass_handoff_node)
1885 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1886 {
1887   return ip4_full_reass_handoff_node_inline (vm, node, frame, NORMAL,
1888                                              true /* is_local */);
1889 }
1890
1891 VLIB_REGISTER_NODE (ip4_local_full_reass_handoff_node) = {
1892   .name = "ip4-local-full-reassembly-handoff",
1893   .vector_size = sizeof (u32),
1894   .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
1895   .error_strings = ip4_full_reass_handoff_error_strings,
1896   .format_trace = format_ip4_full_reass_handoff_trace,
1897
1898   .n_next_nodes = 1,
1899
1900   .next_nodes = {
1901     [0] = "error-drop",
1902   },
1903 };
1904
1905 VLIB_NODE_FN (ip4_full_reass_feature_handoff_node) (vlib_main_t * vm,
1906                                                     vlib_node_runtime_t *
1907                                                     node,
1908                                                     vlib_frame_t * frame)
1909 {
1910   return ip4_full_reass_handoff_node_inline (vm, node, frame, FEATURE,
1911                                              false /* is_local */);
1912 }
1913
1914
1915 VLIB_REGISTER_NODE (ip4_full_reass_feature_handoff_node) = {
1916   .name = "ip4-full-reass-feature-hoff",
1917   .vector_size = sizeof (u32),
1918   .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
1919   .error_strings = ip4_full_reass_handoff_error_strings,
1920   .format_trace = format_ip4_full_reass_handoff_trace,
1921
1922   .n_next_nodes = 1,
1923
1924   .next_nodes = {
1925     [0] = "error-drop",
1926   },
1927 };
1928
1929 VLIB_NODE_FN (ip4_full_reass_custom_handoff_node) (vlib_main_t * vm,
1930                                                     vlib_node_runtime_t *
1931                                                     node,
1932                                                     vlib_frame_t * frame)
1933 {
1934   return ip4_full_reass_handoff_node_inline (vm, node, frame, CUSTOM,
1935                                              false /* is_local */);
1936 }
1937
1938
1939 VLIB_REGISTER_NODE (ip4_full_reass_custom_handoff_node) = {
1940   .name = "ip4-full-reass-custom-hoff",
1941   .vector_size = sizeof (u32),
1942   .n_errors = ARRAY_LEN(ip4_full_reass_handoff_error_strings),
1943   .error_strings = ip4_full_reass_handoff_error_strings,
1944   .format_trace = format_ip4_full_reass_handoff_trace,
1945
1946   .n_next_nodes = 1,
1947
1948   .next_nodes = {
1949     [0] = "error-drop",
1950   },
1951 };
1952
1953 #ifndef CLIB_MARCH_VARIANT
1954 int
1955 ip4_full_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
1956 {
1957   ip4_full_reass_main_t *rm = &ip4_full_reass_main;
1958   vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
1959   if (is_enable)
1960     {
1961       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1962         {
1963           ++rm->feature_use_refcount_per_intf[sw_if_index];
1964           return vnet_feature_enable_disable ("ip4-unicast",
1965                                               "ip4-full-reassembly-feature",
1966                                               sw_if_index, 1, 0, 0);
1967         }
1968       ++rm->feature_use_refcount_per_intf[sw_if_index];
1969     }
1970   else
1971     {
1972       --rm->feature_use_refcount_per_intf[sw_if_index];
1973       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1974         return vnet_feature_enable_disable ("ip4-unicast",
1975                                             "ip4-full-reassembly-feature",
1976                                             sw_if_index, 0, 0, 0);
1977     }
1978   return -1;
1979 }
1980
1981 void
1982 ip4_local_full_reass_enable_disable (int enable)
1983 {
1984   if (enable)
1985     {
1986       ip4_full_reass_main.is_local_reass_enabled = 1;
1987     }
1988   else
1989     {
1990       ip4_full_reass_main.is_local_reass_enabled = 0;
1991     }
1992 }
1993
1994 int
1995 ip4_local_full_reass_enabled ()
1996 {
1997   return ip4_full_reass_main.is_local_reass_enabled;
1998 }
1999
2000 #endif
2001
2002 /*
2003  * fd.io coding-style-patch-verification: ON
2004  *
2005  * Local Variables:
2006  * eval: (c-set-style "gnu")
2007  * End:
2008  */