src/vnet/ip/reass/ip6_full_reass.c

   1 /*
   2  * Copyright (c) 2017 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 /**
  17  * @file
  18  * @brief IPv6 Full Reassembly.
  19  *
  20  * This file contains the source code for IPv6 full reassembly.
  21  */
  22
  23 #include <vppinfra/vec.h>
  24 #include <vnet/vnet.h>
  25 #include <vnet/ip/ip.h>
  26 #include <vppinfra/bihash_48_8.h>
  27 #include <vnet/ip/reass/ip6_full_reass.h>
  28 #include <vnet/ip/ip6_inlines.h>
  29
  30 #define MSEC_PER_SEC 1000
  31 #define IP6_FULL_REASS_TIMEOUT_DEFAULT_MS 100
  32 #define IP6_FULL_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000    // 10 seconds default
  33 #define IP6_FULL_REASS_MAX_REASSEMBLIES_DEFAULT 1024
  34 #define IP6_FULL_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
  35 #define IP6_FULL_REASS_HT_LOAD_FACTOR (0.75)
  36
  37 typedef enum
  38 {
  39   IP6_FULL_REASS_RC_OK,
  40   IP6_FULL_REASS_RC_INTERNAL_ERROR,
  41   IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS,
  42   IP6_FULL_REASS_RC_NO_BUF,
  43   IP6_FULL_REASS_RC_HANDOFF,
  44 } ip6_full_reass_rc_t;
  45
  46 typedef struct
  47 {
  48   union
  49   {
  50     struct
  51     {
  52       ip6_address_t src;
  53       ip6_address_t dst;
  54       u32 xx_id;
  55       u32 frag_id;
  56       u8 unused[7];
  57       u8 proto;
  58     };
  59     u64 as_u64[6];
  60   };
  61 } ip6_full_reass_key_t;
  62
  63 typedef union
  64 {
  65   struct
  66   {
  67     u32 reass_index;
  68     u32 memory_owner_thread_index;
  69   };
  70   u64 as_u64;
  71 } ip6_full_reass_val_t;
  72
  73 typedef union
  74 {
  75   struct
  76   {
  77     ip6_full_reass_key_t k;
  78     ip6_full_reass_val_t v;
  79   };
  80   clib_bihash_kv_48_8_t kv;
  81 } ip6_full_reass_kv_t;
  82
  83
  84 always_inline u32
  85 ip6_full_reass_buffer_get_data_offset (vlib_buffer_t * b)
  86 {
  87   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  88   return vnb->ip.reass.range_first - vnb->ip.reass.fragment_first;
  89 }
  90
  91 always_inline u16
  92 ip6_full_reass_buffer_get_data_len (vlib_buffer_t * b)
  93 {
  94   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  95   return clib_min (vnb->ip.reass.range_last, vnb->ip.reass.fragment_last) -
  96     (vnb->ip.reass.fragment_first +
  97      ip6_full_reass_buffer_get_data_offset (b)) + 1;
  98 }
  99
 100 typedef struct
 101 {
 102   // hash table key
 103   ip6_full_reass_key_t key;
 104   // time when last packet was received
 105   f64 last_heard;
 106   // internal id of this reassembly
 107   u64 id;
 108   // buffer index of first buffer in this reassembly context
 109   u32 first_bi;
 110   // last octet of packet, ~0 until fragment without more_fragments arrives
 111   u32 last_packet_octet;
 112   // length of data collected so far
 113   u32 data_len;
 114   // trace operation counter
 115   u32 trace_op_counter;
 116   // next index - used by custom apps (~0 if not set)
 117   u32 next_index;
 118   // error next index - used by custom apps (~0 if not set)
 119   u32 error_next_index;
 120   // minimum fragment length for this reassembly - used to estimate MTU
 121   u16 min_fragment_length;
 122   // number of fragments for this reassembly
 123   u32 fragments_n;
 124   // thread owning memory for this context (whose pool contains this ctx)
 125   u32 memory_owner_thread_index;
 126   // thread which received fragment with offset 0 and which sends out the
 127   // completed reassembly
 128   u32 sendout_thread_index;
 129 } ip6_full_reass_t;
 130
 131 typedef struct
 132 {
 133   ip6_full_reass_t *pool;
 134   u32 reass_n;
 135   u32 id_counter;
 136   clib_spinlock_t lock;
 137 } ip6_full_reass_per_thread_t;
 138
 139 typedef struct
 140 {
 141   // IPv6 config
 142   u32 timeout_ms;
 143   f64 timeout;
 144   u32 expire_walk_interval_ms;
 145   // maximum number of fragments in one reassembly
 146   u32 max_reass_len;
 147   // maximum number of reassemblies
 148   u32 max_reass_n;
 149
 150   // IPv6 runtime
 151   clib_bihash_48_8_t hash;
 152
 153   // per-thread data
 154   ip6_full_reass_per_thread_t *per_thread_data;
 155
 156   // convenience
 157   vlib_main_t *vlib_main;
 158
 159   // node index of ip6-drop node
 160   u32 ip6_drop_idx;
 161   u32 ip6_icmp_error_idx;
 162   u32 ip6_full_reass_expire_node_idx;
 163
 164   /** Worker handoff */
 165   u32 fq_index;
 166   u32 fq_feature_index;
 167
 168   // reference count for enabling/disabling feature - per interface
 169   u32 *feature_use_refcount_per_intf;
 170 } ip6_full_reass_main_t;
 171
 172 extern ip6_full_reass_main_t ip6_full_reass_main;
 173
 174 #ifndef CLIB_MARCH_VARIANT
 175 ip6_full_reass_main_t ip6_full_reass_main;
 176 #endif /* CLIB_MARCH_VARIANT */
 177
 178 typedef enum
 179 {
 180   IP6_FULL_REASSEMBLY_NEXT_INPUT,
 181   IP6_FULL_REASSEMBLY_NEXT_DROP,
 182   IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR,
 183   IP6_FULL_REASSEMBLY_NEXT_HANDOFF,
 184   IP6_FULL_REASSEMBLY_N_NEXT,
 185 } ip6_full_reass_next_t;
 186
 187 typedef enum
 188 {
 189   RANGE_NEW,
 190   RANGE_OVERLAP,
 191   ICMP_ERROR_RT_EXCEEDED,
 192   ICMP_ERROR_FL_TOO_BIG,
 193   ICMP_ERROR_FL_NOT_MULT_8,
 194   FINALIZE,
 195   HANDOFF,
 196 } ip6_full_reass_trace_operation_e;
 197
 198 typedef struct
 199 {
 200   u16 range_first;
 201   u16 range_last;
 202   u32 range_bi;
 203   i32 data_offset;
 204   u32 data_len;
 205   u32 first_bi;
 206 } ip6_full_reass_range_trace_t;
 207
 208 typedef struct
 209 {
 210   ip6_full_reass_trace_operation_e action;
 211   u32 reass_id;
 212   ip6_full_reass_range_trace_t trace_range;
 213   u32 op_id;
 214   u32 fragment_first;
 215   u32 fragment_last;
 216   u32 total_data_len;
 217   u32 thread_id;
 218   u32 thread_id_to;
 219   bool is_after_handoff;
 220   ip6_header_t ip6_header;
 221   ip6_frag_hdr_t ip6_frag_header;
 222 } ip6_full_reass_trace_t;
 223
 224 static void
 225 ip6_full_reass_trace_details (vlib_main_t * vm, u32 bi,
 226                               ip6_full_reass_range_trace_t * trace)
 227 {
 228   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 229   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 230   trace->range_first = vnb->ip.reass.range_first;
 231   trace->range_last = vnb->ip.reass.range_last;
 232   trace->data_offset = ip6_full_reass_buffer_get_data_offset (b);
 233   trace->data_len = ip6_full_reass_buffer_get_data_len (b);
 234   trace->range_bi = bi;
 235 }
 236
 237 static u8 *
 238 format_ip6_full_reass_range_trace (u8 * s, va_list * args)
 239 {
 240   ip6_full_reass_range_trace_t *trace =
 241     va_arg (*args, ip6_full_reass_range_trace_t *);
 242   s =
 243     format (s, "range: [%u, %u], off %d, len %u, bi %u", trace->range_first,
 244             trace->range_last, trace->data_offset, trace->data_len,
 245             trace->range_bi);
 246   return s;
 247 }
 248
 249 static u8 *
 250 format_ip6_full_reass_trace (u8 * s, va_list * args)
 251 {
 252   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 253   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 254   ip6_full_reass_trace_t *t = va_arg (*args, ip6_full_reass_trace_t *);
 255   u32 indent = 0;
 256   if (~0 != t->reass_id)
 257     {
 258       if (t->is_after_handoff)
 259         {
 260           s =
 261             format (s, "%U\n", format_ip6_header, &t->ip6_header,
 262                     sizeof (t->ip6_header));
 263           s =
 264             format (s, "  %U\n", format_ip6_frag_hdr, &t->ip6_frag_header,
 265                     sizeof (t->ip6_frag_header));
 266           indent = 2;
 267         }
 268       s =
 269         format (s, "%Ureass id: %u, op id: %u, ", format_white_space, indent,
 270                 t->reass_id, t->op_id);
 271       indent = format_get_indent (s);
 272       s = format (s, "first bi: %u, data len: %u, ip/fragment[%u, %u]",
 273                   t->trace_range.first_bi, t->total_data_len,
 274                   t->fragment_first, t->fragment_last);
 275     }
 276   switch (t->action)
 277     {
 278     case RANGE_NEW:
 279       s = format (s, "\n%Unew %U", format_white_space, indent,
 280                   format_ip6_full_reass_range_trace, &t->trace_range);
 281       break;
 282     case RANGE_OVERLAP:
 283       s = format (s, "\n%Uoverlap %U", format_white_space, indent,
 284                   format_ip6_full_reass_range_trace, &t->trace_range);
 285       break;
 286     case ICMP_ERROR_FL_TOO_BIG:
 287       s = format (s, "\n%Uicmp-error - frag_len > 65535 %U",
 288                   format_white_space, indent,
 289                   format_ip6_full_reass_range_trace, &t->trace_range);
 290       break;
 291     case ICMP_ERROR_FL_NOT_MULT_8:
 292       s = format (s, "\n%Uicmp-error - frag_len mod 8 != 0 %U",
 293                   format_white_space, indent,
 294                   format_ip6_full_reass_range_trace, &t->trace_range);
 295       break;
 296     case ICMP_ERROR_RT_EXCEEDED:
 297       s = format (s, "\n%Uicmp-error - reassembly time exceeded",
 298                   format_white_space, indent);
 299       break;
 300     case FINALIZE:
 301       s = format (s, "\n%Ufinalize reassembly", format_white_space, indent);
 302       break;
 303     case HANDOFF:
 304       s =
 305         format (s, "handoff from thread #%u to thread #%u", t->thread_id,
 306                 t->thread_id_to);
 307       break;
 308     }
 309   return s;
 310 }
 311
 312 static void
 313 ip6_full_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
 314                           ip6_full_reass_t * reass, u32 bi,
 315                           ip6_frag_hdr_t * ip6_frag_header,
 316                           ip6_full_reass_trace_operation_e action,
 317                           u32 thread_id_to)
 318 {
 319   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 320   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 321   bool is_after_handoff = false;
 322   if (pool_is_free_index
 323       (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b)))
 324     {
 325       // this buffer's trace is gone
 326       b->flags &= ~VLIB_BUFFER_IS_TRACED;
 327       return;
 328     }
 329   if (vlib_buffer_get_trace_thread (b) != vm->thread_index)
 330     {
 331       is_after_handoff = true;
 332     }
 333   ip6_full_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
 334   t->is_after_handoff = is_after_handoff;
 335   if (t->is_after_handoff)
 336     {
 337       clib_memcpy (&t->ip6_header, vlib_buffer_get_current (b),
 338                    clib_min (sizeof (t->ip6_header), b->current_length));
 339       if (ip6_frag_header)
 340         {
 341           clib_memcpy (&t->ip6_frag_header, ip6_frag_header,
 342                        sizeof (t->ip6_frag_header));
 343         }
 344       else
 345         {
 346           clib_memset (&t->ip6_frag_header, 0, sizeof (t->ip6_frag_header));
 347         }
 348     }
 349   if (reass)
 350     {
 351       t->reass_id = reass->id;
 352       t->op_id = reass->trace_op_counter;
 353       t->trace_range.first_bi = reass->first_bi;
 354       t->total_data_len = reass->data_len;
 355       ++reass->trace_op_counter;
 356     }
 357   else
 358     {
 359       t->reass_id = ~0;
 360     }
 361   t->action = action;
 362   t->thread_id = vm->thread_index;
 363   t->thread_id_to = thread_id_to;
 364   ip6_full_reass_trace_details (vm, bi, &t->trace_range);
 365   t->fragment_first = vnb->ip.reass.fragment_first;
 366   t->fragment_last = vnb->ip.reass.fragment_last;
 367 #if 0
 368   static u8 *s = NULL;
 369   s = format (s, "%U", format_ip6_full_reass_trace, NULL, NULL, t);
 370   printf ("%.*s\n", vec_len (s), s);
 371   fflush (stdout);
 372   vec_reset_length (s);
 373 #endif
 374 }
 375
 376 always_inline void
 377 ip6_full_reass_free_ctx (ip6_full_reass_per_thread_t * rt,
 378                          ip6_full_reass_t * reass)
 379 {
 380   pool_put (rt->pool, reass);
 381   --rt->reass_n;
 382 }
 383
 384 always_inline void
 385 ip6_full_reass_free (ip6_full_reass_main_t * rm,
 386                      ip6_full_reass_per_thread_t * rt,
 387                      ip6_full_reass_t * reass)
 388 {
 389   clib_bihash_kv_48_8_t kv;
 390   kv.key[0] = reass->key.as_u64[0];
 391   kv.key[1] = reass->key.as_u64[1];
 392   kv.key[2] = reass->key.as_u64[2];
 393   kv.key[3] = reass->key.as_u64[3];
 394   kv.key[4] = reass->key.as_u64[4];
 395   kv.key[5] = reass->key.as_u64[5];
 396   clib_bihash_add_del_48_8 (&rm->hash, &kv, 0);
 397   ip6_full_reass_free_ctx (rt, reass);
 398 }
 399
 400 always_inline void
 401 ip6_full_reass_drop_all (vlib_main_t *vm, vlib_node_runtime_t *node,
 402                          ip6_full_reass_t *reass)
 403 {
 404   u32 range_bi = reass->first_bi;
 405   vlib_buffer_t *range_b;
 406   vnet_buffer_opaque_t *range_vnb;
 407   u32 *to_free = NULL;
 408   while (~0 != range_bi)
 409     {
 410       range_b = vlib_get_buffer (vm, range_bi);
 411       range_vnb = vnet_buffer (range_b);
 412       u32 bi = range_bi;
 413       while (~0 != bi)
 414         {
 415           vec_add1 (to_free, bi);
 416           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 417           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 418             {
 419               bi = b->next_buffer;
 420               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 421             }
 422           else
 423             {
 424               bi = ~0;
 425             }
 426         }
 427       range_bi = range_vnb->ip.reass.next_range_bi;
 428     }
 429   /* send to next_error_index */
 430   if (~0 != reass->error_next_index)
 431     {
 432       u32 n_left_to_next, *to_next, next_index;
 433
 434       next_index = reass->error_next_index;
 435       u32 bi = ~0;
 436
 437       while (vec_len (to_free) > 0)
 438         {
 439           vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 440
 441           while (vec_len (to_free) > 0 && n_left_to_next > 0)
 442             {
 443               bi = vec_pop (to_free);
 444
 445               if (~0 != bi)
 446                 {
 447                   to_next[0] = bi;
 448                   to_next += 1;
 449                   n_left_to_next -= 1;
 450                 }
 451             }
 452           vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 453         }
 454     }
 455   else
 456     {
 457       vlib_buffer_free (vm, to_free, vec_len (to_free));
 458     }
 459   vec_free (to_free);
 460 }
 461
 462 always_inline void
 463 ip6_full_reass_on_timeout (vlib_main_t * vm, vlib_node_runtime_t * node,
 464                            ip6_full_reass_t * reass, u32 * icmp_bi)
 465 {
 466   if (~0 == reass->first_bi)
 467     {
 468       return;
 469     }
 470   if (~0 == reass->next_index)  // custom apps don't want icmp
 471     {
 472       vlib_buffer_t *b = vlib_get_buffer (vm, reass->first_bi);
 473       if (0 == vnet_buffer (b)->ip.reass.fragment_first)
 474         {
 475           *icmp_bi = reass->first_bi;
 476           if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
 477             {
 478               ip6_full_reass_add_trace (vm, node, reass, reass->first_bi, NULL,
 479                                         ICMP_ERROR_RT_EXCEEDED, ~0);
 480             }
 481           // fragment with offset zero received - send icmp message back
 482           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 483             {
 484               // separate first buffer from chain and steer it towards icmp node
 485               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 486               reass->first_bi = b->next_buffer;
 487             }
 488           else
 489             {
 490               reass->first_bi = vnet_buffer (b)->ip.reass.next_range_bi;
 491             }
 492           icmp6_error_set_vnet_buffer (b, ICMP6_time_exceeded,
 493                                        ICMP6_time_exceeded_fragment_reassembly_time_exceeded,
 494                                        0);
 495         }
 496     }
 497   ip6_full_reass_drop_all (vm, node, reass);
 498 }
 499
 500 always_inline ip6_full_reass_t *
 501 ip6_full_reass_find_or_create (vlib_main_t *vm, vlib_node_runtime_t *node,
 502                                ip6_full_reass_main_t *rm,
 503                                ip6_full_reass_per_thread_t *rt,
 504                                ip6_full_reass_kv_t *kv, u32 *icmp_bi,
 505                                u8 *do_handoff, int skip_bihash)
 506 {
 507   ip6_full_reass_t *reass;
 508   f64 now;
 509
 510 again:
 511
 512   reass = NULL;
 513   now = vlib_time_now (vm);
 514
 515   if (!skip_bihash && !clib_bihash_search_48_8 (&rm->hash, &kv->kv, &kv->kv))
 516     {
 517       if (vm->thread_index != kv->v.memory_owner_thread_index)
 518         {
 519           *do_handoff = 1;
 520           return NULL;
 521         }
 522
 523       reass =
 524         pool_elt_at_index (rm->per_thread_data
 525                            [kv->v.memory_owner_thread_index].pool,
 526                            kv->v.reass_index);
 527
 528       if (now > reass->last_heard + rm->timeout)
 529         {
 530           ip6_full_reass_on_timeout (vm, node, reass, icmp_bi);
 531           ip6_full_reass_free (rm, rt, reass);
 532           reass = NULL;
 533         }
 534     }
 535
 536   if (reass)
 537     {
 538       reass->last_heard = now;
 539       return reass;
 540     }
 541
 542   if (rt->reass_n >= rm->max_reass_n)
 543     {
 544       reass = NULL;
 545       return reass;
 546     }
 547   else
 548     {
 549       pool_get (rt->pool, reass);
 550       clib_memset (reass, 0, sizeof (*reass));
 551       reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
 552       ++rt->id_counter;
 553       reass->first_bi = ~0;
 554       reass->last_packet_octet = ~0;
 555       reass->data_len = 0;
 556       reass->next_index = ~0;
 557       reass->error_next_index = ~0;
 558       ++rt->reass_n;
 559     }
 560
 561   kv->v.reass_index = (reass - rt->pool);
 562   kv->v.memory_owner_thread_index = vm->thread_index;
 563   reass->last_heard = now;
 564
 565   if (!skip_bihash)
 566     {
 567       reass->key.as_u64[0] = kv->kv.key[0];
 568       reass->key.as_u64[1] = kv->kv.key[1];
 569       reass->key.as_u64[2] = kv->kv.key[2];
 570       reass->key.as_u64[3] = kv->kv.key[3];
 571       reass->key.as_u64[4] = kv->kv.key[4];
 572       reass->key.as_u64[5] = kv->kv.key[5];
 573
 574       int rv = clib_bihash_add_del_48_8 (&rm->hash, &kv->kv, 2);
 575       if (rv)
 576         {
 577           ip6_full_reass_free (rm, rt, reass);
 578           reass = NULL;
 579           // if other worker created a context already work with the other copy
 580           if (-2 == rv)
 581             goto again;
 582         }
 583     }
 584   else
 585     {
 586       reass->key.as_u64[0] = ~0;
 587       reass->key.as_u64[1] = ~0;
 588       reass->key.as_u64[2] = ~0;
 589       reass->key.as_u64[3] = ~0;
 590       reass->key.as_u64[4] = ~0;
 591       reass->key.as_u64[5] = ~0;
 592     }
 593
 594   return reass;
 595 }
 596
 597 always_inline ip6_full_reass_rc_t
 598 ip6_full_reass_finalize (vlib_main_t * vm, vlib_node_runtime_t * node,
 599                          ip6_full_reass_main_t * rm,
 600                          ip6_full_reass_per_thread_t * rt,
 601                          ip6_full_reass_t * reass, u32 * bi0, u32 * next0,
 602                          u32 * error0, bool is_custom_app)
 603 {
 604   *bi0 = reass->first_bi;
 605   *error0 = IP6_ERROR_NONE;
 606   ip6_frag_hdr_t *frag_hdr;
 607   vlib_buffer_t *last_b = NULL;
 608   u32 sub_chain_bi = reass->first_bi;
 609   u32 total_length = 0;
 610   u32 buf_cnt = 0;
 611   u32 dropped_cnt = 0;
 612   u32 *vec_drop_compress = NULL;
 613   ip6_full_reass_rc_t rv = IP6_FULL_REASS_RC_OK;
 614   do
 615     {
 616       u32 tmp_bi = sub_chain_bi;
 617       vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
 618       vnet_buffer_opaque_t *vnb = vnet_buffer (tmp);
 619       if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
 620           !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
 621         {
 622           rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 623           goto free_buffers_and_return;
 624         }
 625
 626       u32 data_len = ip6_full_reass_buffer_get_data_len (tmp);
 627       u32 trim_front = vnet_buffer (tmp)->ip.reass.ip6_frag_hdr_offset +
 628         sizeof (*frag_hdr) + ip6_full_reass_buffer_get_data_offset (tmp);
 629       u32 trim_end =
 630         vlib_buffer_length_in_chain (vm, tmp) - trim_front - data_len;
 631       if (tmp_bi == reass->first_bi)
 632         {
 633           /* first buffer - keep ip6 header */
 634           if (0 != ip6_full_reass_buffer_get_data_offset (tmp))
 635             {
 636               rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 637               goto free_buffers_and_return;
 638             }
 639           trim_front = 0;
 640           trim_end = vlib_buffer_length_in_chain (vm, tmp) - data_len -
 641             (vnet_buffer (tmp)->ip.reass.ip6_frag_hdr_offset +
 642              sizeof (*frag_hdr));
 643           if (!(vlib_buffer_length_in_chain (vm, tmp) - trim_end > 0))
 644             {
 645               rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 646               goto free_buffers_and_return;
 647             }
 648         }
 649       u32 keep_data =
 650         vlib_buffer_length_in_chain (vm, tmp) - trim_front - trim_end;
 651       while (1)
 652         {
 653           ++buf_cnt;
 654           if (trim_front)
 655             {
 656               if (trim_front > tmp->current_length)
 657                 {
 658                   /* drop whole buffer */
 659                   vec_add1 (vec_drop_compress, tmp_bi);
 660                   trim_front -= tmp->current_length;
 661                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 662                     {
 663                       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 664                       goto free_buffers_and_return;
 665                     }
 666                   tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 667                   tmp_bi = tmp->next_buffer;
 668                   tmp = vlib_get_buffer (vm, tmp_bi);
 669                   continue;
 670                 }
 671               else
 672                 {
 673                   vlib_buffer_advance (tmp, trim_front);
 674                   trim_front = 0;
 675                 }
 676             }
 677           if (keep_data)
 678             {
 679               if (last_b)
 680                 {
 681                   last_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
 682                   last_b->next_buffer = tmp_bi;
 683                 }
 684               last_b = tmp;
 685               if (keep_data <= tmp->current_length)
 686                 {
 687                   tmp->current_length = keep_data;
 688                   keep_data = 0;
 689                 }
 690               else
 691                 {
 692                   keep_data -= tmp->current_length;
 693                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 694                     {
 695                       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 696                       goto free_buffers_and_return;
 697                     }
 698                 }
 699               total_length += tmp->current_length;
 700             }
 701           else
 702             {
 703               vec_add1 (vec_drop_compress, tmp_bi);
 704               if (reass->first_bi == tmp_bi)
 705                 {
 706                   rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 707                   goto free_buffers_and_return;
 708                 }
 709               ++dropped_cnt;
 710             }
 711           if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
 712             {
 713               tmp_bi = tmp->next_buffer;
 714               tmp = vlib_get_buffer (vm, tmp->next_buffer);
 715             }
 716           else
 717             {
 718               break;
 719             }
 720         }
 721       sub_chain_bi =
 722         vnet_buffer (vlib_get_buffer (vm, sub_chain_bi))->ip.
 723         reass.next_range_bi;
 724     }
 725   while (~0 != sub_chain_bi);
 726
 727   if (!last_b)
 728     {
 729       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 730       goto free_buffers_and_return;
 731     }
 732   last_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 733   vlib_buffer_t *first_b = vlib_get_buffer (vm, reass->first_bi);
 734   if (total_length < first_b->current_length)
 735     {
 736       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 737       goto free_buffers_and_return;
 738     }
 739   total_length -= first_b->current_length;
 740   first_b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 741   first_b->total_length_not_including_first_buffer = total_length;
 742   // drop fragment header
 743   vnet_buffer_opaque_t *first_b_vnb = vnet_buffer (first_b);
 744   ip6_header_t *ip = vlib_buffer_get_current (first_b);
 745   u16 ip6_frag_hdr_offset = first_b_vnb->ip.reass.ip6_frag_hdr_offset;
 746   ip6_ext_hdr_chain_t hdr_chain;
 747   ip6_ext_header_t *prev_hdr = 0;
 748   int res = ip6_ext_header_walk (first_b, ip, IP_PROTOCOL_IPV6_FRAGMENTATION,
 749                                  &hdr_chain);
 750   if (res < 0 ||
 751       (hdr_chain.eh[res].protocol != IP_PROTOCOL_IPV6_FRAGMENTATION))
 752     {
 753       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 754       goto free_buffers_and_return;
 755     }
 756   frag_hdr = ip6_ext_next_header_offset (ip, hdr_chain.eh[res].offset);
 757   if (res > 0)
 758     {
 759       prev_hdr = ip6_ext_next_header_offset (ip, hdr_chain.eh[res - 1].offset);
 760       prev_hdr->next_hdr = frag_hdr->next_hdr;
 761     }
 762   else
 763     {
 764       ip->protocol = frag_hdr->next_hdr;
 765     }
 766   if (hdr_chain.eh[res].offset != ip6_frag_hdr_offset)
 767     {
 768       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 769       goto free_buffers_and_return;
 770     }
 771   memmove (frag_hdr, (u8 *) frag_hdr + sizeof (*frag_hdr),
 772            first_b->current_length - ip6_frag_hdr_offset -
 773            sizeof (ip6_frag_hdr_t));
 774   first_b->current_length -= sizeof (*frag_hdr);
 775   ip->payload_length =
 776     clib_host_to_net_u16 (total_length + first_b->current_length -
 777                           sizeof (*ip));
 778   if (!vlib_buffer_chain_linearize (vm, first_b))
 779     {
 780       rv = IP6_FULL_REASS_RC_NO_BUF;
 781       goto free_buffers_and_return;
 782     }
 783   first_b->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
 784   if (PREDICT_FALSE (first_b->flags & VLIB_BUFFER_IS_TRACED))
 785     {
 786       ip6_full_reass_add_trace (vm, node, reass, reass->first_bi, NULL,
 787                                 FINALIZE, ~0);
 788 #if 0
 789       // following code does a hexdump of packet fragments to stdout ...
 790       do
 791         {
 792           u32 bi = reass->first_bi;
 793           u8 *s = NULL;
 794           while (~0 != bi)
 795             {
 796               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 797               s = format (s, "%u: %U\n", bi, format_hexdump,
 798                           vlib_buffer_get_current (b), b->current_length);
 799               if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 800                 {
 801                   bi = b->next_buffer;
 802                 }
 803               else
 804                 {
 805                   break;
 806                 }
 807             }
 808           printf ("%.*s\n", vec_len (s), s);
 809           fflush (stdout);
 810           vec_free (s);
 811         }
 812       while (0);
 813 #endif
 814     }
 815   if (!is_custom_app)
 816     {
 817       *next0 = IP6_FULL_REASSEMBLY_NEXT_INPUT;
 818     }
 819   else
 820     {
 821       *next0 = reass->next_index;
 822     }
 823   vnet_buffer (first_b)->ip.reass.estimated_mtu = reass->min_fragment_length;
 824   ip6_full_reass_free (rm, rt, reass);
 825   reass = NULL;
 826 free_buffers_and_return:
 827   vlib_buffer_free (vm, vec_drop_compress, vec_len (vec_drop_compress));
 828   vec_free (vec_drop_compress);
 829   return rv;
 830 }
 831
 832 always_inline void
 833 ip6_full_reass_insert_range_in_chain (vlib_main_t * vm,
 834                                       ip6_full_reass_t * reass,
 835                                       u32 prev_range_bi, u32 new_next_bi)
 836 {
 837
 838   vlib_buffer_t *new_next_b = vlib_get_buffer (vm, new_next_bi);
 839   vnet_buffer_opaque_t *new_next_vnb = vnet_buffer (new_next_b);
 840   if (~0 != prev_range_bi)
 841     {
 842       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 843       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 844       new_next_vnb->ip.reass.next_range_bi = prev_vnb->ip.reass.next_range_bi;
 845       prev_vnb->ip.reass.next_range_bi = new_next_bi;
 846     }
 847   else
 848     {
 849       if (~0 != reass->first_bi)
 850         {
 851           new_next_vnb->ip.reass.next_range_bi = reass->first_bi;
 852         }
 853       reass->first_bi = new_next_bi;
 854     }
 855   reass->data_len += ip6_full_reass_buffer_get_data_len (new_next_b);
 856 }
 857
 858 always_inline ip6_full_reass_rc_t
 859 ip6_full_reass_update (vlib_main_t *vm, vlib_node_runtime_t *node,
 860                        ip6_full_reass_main_t *rm,
 861                        ip6_full_reass_per_thread_t *rt,
 862                        ip6_full_reass_t *reass, u32 *bi0, u32 *next0,
 863                        u32 *error0, ip6_frag_hdr_t *frag_hdr,
 864                        bool is_custom_app, u32 *handoff_thread_idx,
 865                        int skip_bihash)
 866 {
 867   int consumed = 0;
 868   vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
 869   vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
 870   if (is_custom_app)
 871     {
 872       reass->next_index = fvnb->ip.reass.next_index;    // store next_index before it's overwritten
 873       reass->error_next_index = fvnb->ip.reass.error_next_index;        // store error_next_index before it is overwritten
 874     }
 875
 876   fvnb->ip.reass.ip6_frag_hdr_offset =
 877     (u8 *) frag_hdr - (u8 *) vlib_buffer_get_current (fb);
 878   ip6_header_t *fip = vlib_buffer_get_current (fb);
 879   if (fb->current_length < sizeof (*fip) ||
 880       fvnb->ip.reass.ip6_frag_hdr_offset == 0 ||
 881       fvnb->ip.reass.ip6_frag_hdr_offset >= fb->current_length)
 882     {
 883       return IP6_FULL_REASS_RC_INTERNAL_ERROR;
 884     }
 885
 886   u32 fragment_first = fvnb->ip.reass.fragment_first =
 887     ip6_frag_hdr_offset_bytes (frag_hdr);
 888   u32 fragment_length =
 889     vlib_buffer_length_in_chain (vm, fb) -
 890     (fvnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
 891   u32 fragment_last = fvnb->ip.reass.fragment_last =
 892     fragment_first + fragment_length - 1;
 893   int more_fragments = ip6_frag_hdr_more (frag_hdr);
 894   u32 candidate_range_bi = reass->first_bi;
 895   u32 prev_range_bi = ~0;
 896   fvnb->ip.reass.range_first = fragment_first;
 897   fvnb->ip.reass.range_last = fragment_last;
 898   fvnb->ip.reass.next_range_bi = ~0;
 899   if (!more_fragments)
 900     {
 901       reass->last_packet_octet = fragment_last;
 902     }
 903   if (~0 == reass->first_bi)
 904     {
 905       // starting a new reassembly
 906       ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi, *bi0);
 907       reass->min_fragment_length = clib_net_to_host_u16 (fip->payload_length);
 908       consumed = 1;
 909       reass->fragments_n = 1;
 910       goto check_if_done_maybe;
 911     }
 912   reass->min_fragment_length =
 913     clib_min (clib_net_to_host_u16 (fip->payload_length),
 914               fvnb->ip.reass.estimated_mtu);
 915   while (~0 != candidate_range_bi)
 916     {
 917       vlib_buffer_t *candidate_b = vlib_get_buffer (vm, candidate_range_bi);
 918       vnet_buffer_opaque_t *candidate_vnb = vnet_buffer (candidate_b);
 919       if (fragment_first > candidate_vnb->ip.reass.range_last)
 920         {
 921           // this fragments starts after candidate range
 922           prev_range_bi = candidate_range_bi;
 923           candidate_range_bi = candidate_vnb->ip.reass.next_range_bi;
 924           if (candidate_vnb->ip.reass.range_last < fragment_last &&
 925               ~0 == candidate_range_bi)
 926             {
 927               // special case - this fragment falls beyond all known ranges
 928               ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
 929                                                     *bi0);
 930               consumed = 1;
 931               break;
 932             }
 933           continue;
 934         }
 935       if (fragment_last < candidate_vnb->ip.reass.range_first)
 936         {
 937           // this fragment ends before candidate range without any overlap
 938           ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
 939                                                 *bi0);
 940           consumed = 1;
 941         }
 942       else if (fragment_first == candidate_vnb->ip.reass.range_first &&
 943                fragment_last == candidate_vnb->ip.reass.range_last)
 944         {
 945           // duplicate fragment - ignore
 946         }
 947       else
 948         {
 949           // overlapping fragment - not allowed by RFC 8200
 950           if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 951             {
 952               ip6_full_reass_add_trace (vm, node, reass, *bi0, frag_hdr,
 953                                         RANGE_OVERLAP, ~0);
 954             }
 955           ip6_full_reass_drop_all (vm, node, reass);
 956           ip6_full_reass_free (rm, rt, reass);
 957           *next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
 958           *error0 = IP6_ERROR_REASS_OVERLAPPING_FRAGMENT;
 959           return IP6_FULL_REASS_RC_OK;
 960         }
 961       break;
 962     }
 963   ++reass->fragments_n;
 964 check_if_done_maybe:
 965   if (consumed)
 966     {
 967       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 968         {
 969           ip6_full_reass_add_trace (vm, node, reass, *bi0, frag_hdr, RANGE_NEW,
 970                                     ~0);
 971         }
 972     }
 973   else if (skip_bihash)
 974     {
 975       // if this reassembly is not in bihash, then the packet must have been
 976       // consumed
 977       return IP6_FULL_REASS_RC_INTERNAL_ERROR;
 978     }
 979   if (~0 != reass->last_packet_octet &&
 980       reass->data_len == reass->last_packet_octet + 1)
 981     {
 982       *handoff_thread_idx = reass->sendout_thread_index;
 983       int handoff =
 984         reass->memory_owner_thread_index != reass->sendout_thread_index;
 985       ip6_full_reass_rc_t rc =
 986         ip6_full_reass_finalize (vm, node, rm, rt, reass, bi0, next0, error0,
 987                                  is_custom_app);
 988       if (IP6_FULL_REASS_RC_OK == rc && handoff)
 989         {
 990           return IP6_FULL_REASS_RC_HANDOFF;
 991         }
 992       return rc;
 993     }
 994   else
 995     {
 996       if (skip_bihash)
 997         {
 998           // if this reassembly is not in bihash, it should've been an atomic
 999           // fragment and thus finalized
1000           return IP6_FULL_REASS_RC_INTERNAL_ERROR;
1001         }
1002       if (consumed)
1003         {
1004           *bi0 = ~0;
1005           if (reass->fragments_n > rm->max_reass_len)
1006             {
1007               return IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS;
1008             }
1009         }
1010       else
1011         {
1012           *next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1013           *error0 = IP6_ERROR_REASS_DUPLICATE_FRAGMENT;
1014         }
1015     }
1016   return IP6_FULL_REASS_RC_OK;
1017 }
1018
1019 always_inline bool
1020 ip6_full_reass_verify_upper_layer_present (vlib_node_runtime_t *node,
1021                                            vlib_buffer_t *b,
1022                                            ip6_ext_hdr_chain_t *hc)
1023 {
1024   int nh = hc->eh[hc->length - 1].protocol;
1025   /* Checking to see if it's a terminating header */
1026   if (ip6_ext_hdr (nh))
1027     {
1028       icmp6_error_set_vnet_buffer (
1029         b, ICMP6_parameter_problem,
1030         ICMP6_parameter_problem_first_fragment_has_incomplete_header_chain, 0);
1031       b->error = node->errors[IP6_ERROR_REASS_MISSING_UPPER];
1032       return false;
1033     }
1034   return true;
1035 }
1036
1037 always_inline bool
1038 ip6_full_reass_verify_fragment_multiple_8 (vlib_main_t * vm,
1039                                            vlib_buffer_t * b,
1040                                            ip6_frag_hdr_t * frag_hdr)
1041 {
1042   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1043   ip6_header_t *ip = vlib_buffer_get_current (b);
1044   int more_fragments = ip6_frag_hdr_more (frag_hdr);
1045   u32 fragment_length =
1046     vlib_buffer_length_in_chain (vm, b) -
1047     (vnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
1048   if (more_fragments && 0 != fragment_length % 8)
1049     {
1050       icmp6_error_set_vnet_buffer (b, ICMP6_parameter_problem,
1051                                    ICMP6_parameter_problem_erroneous_header_field,
1052                                    (u8 *) & ip->payload_length - (u8 *) ip);
1053       return false;
1054     }
1055   return true;
1056 }
1057
1058 always_inline bool
1059 ip6_full_reass_verify_packet_size_lt_64k (vlib_main_t * vm,
1060                                           vlib_buffer_t * b,
1061                                           ip6_frag_hdr_t * frag_hdr)
1062 {
1063   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1064   u32 fragment_first = ip6_frag_hdr_offset_bytes (frag_hdr);
1065   u32 fragment_length =
1066     vlib_buffer_length_in_chain (vm, b) -
1067     (vnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
1068   if (fragment_first + fragment_length > 65535)
1069     {
1070       ip6_header_t *ip0 = vlib_buffer_get_current (b);
1071       icmp6_error_set_vnet_buffer (b, ICMP6_parameter_problem,
1072                                    ICMP6_parameter_problem_erroneous_header_field,
1073                                    (u8 *) & frag_hdr->fragment_offset_and_more
1074                                    - (u8 *) ip0);
1075       return false;
1076     }
1077   return true;
1078 }
1079
1080 always_inline uword
1081 ip6_full_reassembly_inline (vlib_main_t * vm,
1082                             vlib_node_runtime_t * node,
1083                             vlib_frame_t * frame, bool is_feature,
1084                             bool is_custom_app)
1085 {
1086   u32 *from = vlib_frame_vector_args (frame);
1087   u32 n_left_from, n_left_to_next, *to_next, next_index;
1088   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1089   ip6_full_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
1090   clib_spinlock_lock (&rt->lock);
1091
1092   n_left_from = frame->n_vectors;
1093   next_index = node->cached_next_index;
1094   while (n_left_from > 0)
1095     {
1096       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1097
1098       while (n_left_from > 0 && n_left_to_next > 0)
1099         {
1100           u32 bi0;
1101           vlib_buffer_t *b0;
1102           u32 next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1103           u32 error0 = IP6_ERROR_NONE;
1104           u32 icmp_bi = ~0;
1105
1106           bi0 = from[0];
1107           b0 = vlib_get_buffer (vm, bi0);
1108
1109           ip6_header_t *ip0 = vlib_buffer_get_current (b0);
1110           ip6_frag_hdr_t *frag_hdr;
1111           ip6_ext_hdr_chain_t hdr_chain;
1112           int res = ip6_ext_header_walk (
1113             b0, ip0, IP_PROTOCOL_IPV6_FRAGMENTATION, &hdr_chain);
1114           if (res < 0 ||
1115               hdr_chain.eh[res].protocol != IP_PROTOCOL_IPV6_FRAGMENTATION)
1116             {
1117               // this is a mangled packet - no fragmentation
1118               next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1119               goto skip_reass;
1120             }
1121           frag_hdr =
1122             ip6_ext_next_header_offset (ip0, hdr_chain.eh[res].offset);
1123           vnet_buffer (b0)->ip.reass.ip6_frag_hdr_offset =
1124             hdr_chain.eh[res].offset;
1125
1126           if (0 == ip6_frag_hdr_offset (frag_hdr))
1127             {
1128               // first fragment - verify upper-layer is present
1129               if (!ip6_full_reass_verify_upper_layer_present (node, b0,
1130                                                               &hdr_chain))
1131                 {
1132                   next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1133                   goto skip_reass;
1134                 }
1135             }
1136           if (!ip6_full_reass_verify_fragment_multiple_8 (vm, b0, frag_hdr) ||
1137               !ip6_full_reass_verify_packet_size_lt_64k (vm, b0, frag_hdr))
1138             {
1139               next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1140               goto skip_reass;
1141             }
1142
1143           int skip_bihash = 0;
1144           ip6_full_reass_kv_t kv;
1145           u8 do_handoff = 0;
1146
1147           if (0 == ip6_frag_hdr_offset (frag_hdr) &&
1148               !ip6_frag_hdr_more (frag_hdr))
1149             {
1150               // this is atomic fragment and needs to be processed separately
1151               skip_bihash = 1;
1152             }
1153           else
1154             {
1155               kv.k.as_u64[0] = ip0->src_address.as_u64[0];
1156               kv.k.as_u64[1] = ip0->src_address.as_u64[1];
1157               kv.k.as_u64[2] = ip0->dst_address.as_u64[0];
1158               kv.k.as_u64[3] = ip0->dst_address.as_u64[1];
1159               kv.k.as_u64[4] =
1160                 ((u64) vec_elt (ip6_main.fib_index_by_sw_if_index,
1161                                 vnet_buffer (b0)->sw_if_index[VLIB_RX]))
1162                   << 32 |
1163                 (u64) frag_hdr->identification;
1164               kv.k.as_u64[5] = ip0->protocol;
1165             }
1166
1167           ip6_full_reass_t *reass = ip6_full_reass_find_or_create (
1168             vm, node, rm, rt, &kv, &icmp_bi, &do_handoff, skip_bihash);
1169
1170           if (reass)
1171             {
1172               const u32 fragment_first = ip6_frag_hdr_offset (frag_hdr);
1173               if (0 == fragment_first)
1174                 {
1175                   reass->sendout_thread_index = vm->thread_index;
1176                 }
1177             }
1178           if (PREDICT_FALSE (do_handoff))
1179             {
1180               next0 = IP6_FULL_REASSEMBLY_NEXT_HANDOFF;
1181               vnet_buffer (b0)->ip.reass.owner_thread_index =
1182                 kv.v.memory_owner_thread_index;
1183             }
1184           else if (reass)
1185             {
1186               u32 handoff_thread_idx;
1187               u32 counter = ~0;
1188               switch (ip6_full_reass_update (
1189                 vm, node, rm, rt, reass, &bi0, &next0, &error0, frag_hdr,
1190                 is_custom_app, &handoff_thread_idx, skip_bihash))
1191                 {
1192                 case IP6_FULL_REASS_RC_OK:
1193                   /* nothing to do here */
1194                   break;
1195                 case IP6_FULL_REASS_RC_HANDOFF:
1196                   next0 = IP6_FULL_REASSEMBLY_NEXT_HANDOFF;
1197                   b0 = vlib_get_buffer (vm, bi0);
1198                   vnet_buffer (b0)->ip.reass.owner_thread_index =
1199                     handoff_thread_idx;
1200                   break;
1201                 case IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS:
1202                   counter = IP6_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG;
1203                   break;
1204                 case IP6_FULL_REASS_RC_NO_BUF:
1205                   counter = IP6_ERROR_REASS_NO_BUF;
1206                   break;
1207                 case IP6_FULL_REASS_RC_INTERNAL_ERROR:
1208                   counter = IP6_ERROR_REASS_INTERNAL_ERROR;
1209                   break;
1210                 }
1211               if (~0 != counter)
1212                 {
1213                   vlib_node_increment_counter (vm, node->node_index, counter,
1214                                                1);
1215                   ip6_full_reass_drop_all (vm, node, reass);
1216                   ip6_full_reass_free (rm, rt, reass);
1217                   goto next_packet;
1218                 }
1219             }
1220           else
1221             {
1222               if (is_feature)
1223                 {
1224                   next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1225                 }
1226               else
1227                 {
1228                   vnet_buffer_opaque_t *fvnb = vnet_buffer (b0);
1229                   next0 = fvnb->ip.reass.error_next_index;
1230                 }
1231               error0 = IP6_ERROR_REASS_LIMIT_REACHED;
1232             }
1233
1234           if (~0 != bi0)
1235             {
1236             skip_reass:
1237               to_next[0] = bi0;
1238               to_next += 1;
1239               n_left_to_next -= 1;
1240
1241               /* bi0 might have been updated by reass_finalize, reload */
1242               b0 = vlib_get_buffer (vm, bi0);
1243               if (IP6_ERROR_NONE != error0)
1244                 {
1245                   b0->error = node->errors[error0];
1246                 }
1247
1248               if (next0 == IP6_FULL_REASSEMBLY_NEXT_HANDOFF)
1249                 {
1250                   if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1251                     {
1252                       ip6_full_reass_add_trace (
1253                         vm, node, NULL, bi0, frag_hdr, HANDOFF,
1254                         vnet_buffer (b0)->ip.reass.owner_thread_index);
1255                     }
1256                 }
1257               else if (is_feature && IP6_ERROR_NONE == error0)
1258                 {
1259                   vnet_feature_next (&next0, b0);
1260                 }
1261               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1262                                                n_left_to_next, bi0, next0);
1263             }
1264
1265           if (~0 != icmp_bi)
1266             {
1267               next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1268               to_next[0] = icmp_bi;
1269               to_next += 1;
1270               n_left_to_next -= 1;
1271               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1272                                                n_left_to_next, icmp_bi,
1273                                                next0);
1274             }
1275         next_packet:
1276           from += 1;
1277           n_left_from -= 1;
1278         }
1279
1280       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1281     }
1282
1283   clib_spinlock_unlock (&rt->lock);
1284   return frame->n_vectors;
1285 }
1286
1287 static char *ip6_full_reassembly_error_strings[] = {
1288 #define _(sym, string) string,
1289   foreach_ip6_error
1290 #undef _
1291 };
1292
1293 VLIB_NODE_FN (ip6_full_reass_node) (vlib_main_t * vm,
1294                                     vlib_node_runtime_t * node,
1295                                     vlib_frame_t * frame)
1296 {
1297   return ip6_full_reassembly_inline (vm, node, frame, false /* is_feature */ ,
1298                                      false /* is_custom_app */ );
1299 }
1300
1301 VLIB_REGISTER_NODE (ip6_full_reass_node) = {
1302     .name = "ip6-full-reassembly",
1303     .vector_size = sizeof (u32),
1304     .format_trace = format_ip6_full_reass_trace,
1305     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1306     .error_strings = ip6_full_reassembly_error_strings,
1307     .n_next_nodes = IP6_FULL_REASSEMBLY_N_NEXT,
1308     .next_nodes =
1309         {
1310                 [IP6_FULL_REASSEMBLY_NEXT_INPUT] = "ip6-input",
1311                 [IP6_FULL_REASSEMBLY_NEXT_DROP] = "ip6-drop",
1312                 [IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR] = "ip6-icmp-error",
1313                 [IP6_FULL_REASSEMBLY_NEXT_HANDOFF] = "ip6-full-reassembly-handoff",
1314         },
1315 };
1316
1317 VLIB_NODE_FN (ip6_full_reass_node_feature) (vlib_main_t * vm,
1318                                             vlib_node_runtime_t * node,
1319                                             vlib_frame_t * frame)
1320 {
1321   return ip6_full_reassembly_inline (vm, node, frame, true /* is_feature */ ,
1322                                      false /* is_custom_app */ );
1323 }
1324
1325 VLIB_REGISTER_NODE (ip6_full_reass_node_feature) = {
1326     .name = "ip6-full-reassembly-feature",
1327     .vector_size = sizeof (u32),
1328     .format_trace = format_ip6_full_reass_trace,
1329     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1330     .error_strings = ip6_full_reassembly_error_strings,
1331     .n_next_nodes = IP6_FULL_REASSEMBLY_N_NEXT,
1332     .next_nodes =
1333         {
1334                 [IP6_FULL_REASSEMBLY_NEXT_INPUT] = "ip6-input",
1335                 [IP6_FULL_REASSEMBLY_NEXT_DROP] = "ip6-drop",
1336                 [IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR] = "ip6-icmp-error",
1337                 [IP6_FULL_REASSEMBLY_NEXT_HANDOFF] = "ip6-full-reass-feature-hoff",
1338         },
1339 };
1340
1341 VNET_FEATURE_INIT (ip6_full_reassembly_feature, static) = {
1342     .arc_name = "ip6-unicast",
1343     .node_name = "ip6-full-reassembly-feature",
1344     .runs_before = VNET_FEATURES ("ip6-lookup",
1345                                   "ipsec6-input-feature"),
1346     .runs_after = 0,
1347 };
1348
1349 #ifndef CLIB_MARCH_VARIANT
1350 static u32
1351 ip6_full_reass_get_nbuckets ()
1352 {
1353   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1354   u32 nbuckets;
1355   u8 i;
1356
1357   nbuckets = (u32) (rm->max_reass_n / IP6_FULL_REASS_HT_LOAD_FACTOR);
1358
1359   for (i = 0; i < 31; i++)
1360     if ((1 << i) >= nbuckets)
1361       break;
1362   nbuckets = 1 << i;
1363
1364   return nbuckets;
1365 }
1366 #endif /* CLIB_MARCH_VARIANT */
1367
1368 typedef enum
1369 {
1370   IP6_EVENT_CONFIG_CHANGED = 1,
1371 } ip6_full_reass_event_t;
1372
1373 #ifndef CLIB_MARCH_VARIANT
1374 typedef struct
1375 {
1376   int failure;
1377   clib_bihash_48_8_t *new_hash;
1378 } ip6_rehash_cb_ctx;
1379
1380 static int
1381 ip6_rehash_cb (clib_bihash_kv_48_8_t * kv, void *_ctx)
1382 {
1383   ip6_rehash_cb_ctx *ctx = _ctx;
1384   if (clib_bihash_add_del_48_8 (ctx->new_hash, kv, 1))
1385     {
1386       ctx->failure = 1;
1387     }
1388   return (BIHASH_WALK_CONTINUE);
1389 }
1390
1391 static void
1392 ip6_full_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
1393                            u32 max_reassembly_length,
1394                            u32 expire_walk_interval_ms)
1395 {
1396   ip6_full_reass_main.timeout_ms = timeout_ms;
1397   ip6_full_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
1398   ip6_full_reass_main.max_reass_n = max_reassemblies;
1399   ip6_full_reass_main.max_reass_len = max_reassembly_length;
1400   ip6_full_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
1401 }
1402
1403 vnet_api_error_t
1404 ip6_full_reass_set (u32 timeout_ms, u32 max_reassemblies,
1405                     u32 max_reassembly_length, u32 expire_walk_interval_ms)
1406 {
1407   u32 old_nbuckets = ip6_full_reass_get_nbuckets ();
1408   ip6_full_reass_set_params (timeout_ms, max_reassemblies,
1409                              max_reassembly_length, expire_walk_interval_ms);
1410   vlib_process_signal_event (ip6_full_reass_main.vlib_main,
1411                              ip6_full_reass_main.ip6_full_reass_expire_node_idx,
1412                              IP6_EVENT_CONFIG_CHANGED, 0);
1413   u32 new_nbuckets = ip6_full_reass_get_nbuckets ();
1414   if (ip6_full_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
1415     {
1416       clib_bihash_48_8_t new_hash;
1417       clib_memset (&new_hash, 0, sizeof (new_hash));
1418       ip6_rehash_cb_ctx ctx;
1419       ctx.failure = 0;
1420       ctx.new_hash = &new_hash;
1421       clib_bihash_init_48_8 (&new_hash, "ip6-full-reass", new_nbuckets,
1422                              new_nbuckets * 1024);
1423       clib_bihash_foreach_key_value_pair_48_8 (&ip6_full_reass_main.hash,
1424                                                ip6_rehash_cb, &ctx);
1425       if (ctx.failure)
1426         {
1427           clib_bihash_free_48_8 (&new_hash);
1428           return -1;
1429         }
1430       else
1431         {
1432           clib_bihash_free_48_8 (&ip6_full_reass_main.hash);
1433           clib_memcpy_fast (&ip6_full_reass_main.hash, &new_hash,
1434                             sizeof (ip6_full_reass_main.hash));
1435           clib_bihash_copied (&ip6_full_reass_main.hash, &new_hash);
1436         }
1437     }
1438   return 0;
1439 }
1440
1441 vnet_api_error_t
1442 ip6_full_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
1443                     u32 * max_reassembly_length,
1444                     u32 * expire_walk_interval_ms)
1445 {
1446   *timeout_ms = ip6_full_reass_main.timeout_ms;
1447   *max_reassemblies = ip6_full_reass_main.max_reass_n;
1448   *max_reassembly_length = ip6_full_reass_main.max_reass_len;
1449   *expire_walk_interval_ms = ip6_full_reass_main.expire_walk_interval_ms;
1450   return 0;
1451 }
1452
1453 static clib_error_t *
1454 ip6_full_reass_init_function (vlib_main_t * vm)
1455 {
1456   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1457   clib_error_t *error = 0;
1458   u32 nbuckets;
1459   vlib_node_t *node;
1460
1461   rm->vlib_main = vm;
1462
1463   vec_validate (rm->per_thread_data, vlib_num_workers ());
1464   ip6_full_reass_per_thread_t *rt;
1465   vec_foreach (rt, rm->per_thread_data)
1466   {
1467     clib_spinlock_init (&rt->lock);
1468     pool_alloc (rt->pool, rm->max_reass_n);
1469   }
1470
1471   node = vlib_get_node_by_name (vm, (u8 *) "ip6-full-reassembly-expire-walk");
1472   ASSERT (node);
1473   rm->ip6_full_reass_expire_node_idx = node->index;
1474
1475   ip6_full_reass_set_params (IP6_FULL_REASS_TIMEOUT_DEFAULT_MS,
1476                              IP6_FULL_REASS_MAX_REASSEMBLIES_DEFAULT,
1477                              IP6_FULL_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
1478                              IP6_FULL_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
1479
1480   nbuckets = ip6_full_reass_get_nbuckets ();
1481   clib_bihash_init_48_8 (&rm->hash, "ip6-full-reass", nbuckets,
1482                          nbuckets * 1024);
1483
1484   node = vlib_get_node_by_name (vm, (u8 *) "ip6-drop");
1485   ASSERT (node);
1486   rm->ip6_drop_idx = node->index;
1487   node = vlib_get_node_by_name (vm, (u8 *) "ip6-icmp-error");
1488   ASSERT (node);
1489   rm->ip6_icmp_error_idx = node->index;
1490
1491   if ((error = vlib_call_init_function (vm, ip_main_init)))
1492     return error;
1493   ip6_register_protocol (IP_PROTOCOL_IPV6_FRAGMENTATION,
1494                          ip6_full_reass_node.index);
1495
1496   rm->fq_index = vlib_frame_queue_main_init (ip6_full_reass_node.index, 0);
1497   rm->fq_feature_index =
1498     vlib_frame_queue_main_init (ip6_full_reass_node_feature.index, 0);
1499
1500   rm->feature_use_refcount_per_intf = NULL;
1501   return error;
1502 }
1503
1504 VLIB_INIT_FUNCTION (ip6_full_reass_init_function);
1505 #endif /* CLIB_MARCH_VARIANT */
1506
1507 static uword
1508 ip6_full_reass_walk_expired (vlib_main_t *vm, vlib_node_runtime_t *node,
1509                              CLIB_UNUSED (vlib_frame_t *f))
1510 {
1511   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1512   uword event_type, *event_data = 0;
1513
1514   while (true)
1515     {
1516       vlib_process_wait_for_event_or_clock (vm,
1517                                             (f64) rm->expire_walk_interval_ms
1518                                             / (f64) MSEC_PER_SEC);
1519       event_type = vlib_process_get_events (vm, &event_data);
1520
1521       switch (event_type)
1522         {
1523         case ~0:
1524           /* no events => timeout */
1525           /* fallthrough */
1526         case IP6_EVENT_CONFIG_CHANGED:
1527           /* nothing to do here */
1528           break;
1529         default:
1530           clib_warning ("BUG: event type 0x%wx", event_type);
1531           break;
1532         }
1533       f64 now = vlib_time_now (vm);
1534
1535       ip6_full_reass_t *reass;
1536       int *pool_indexes_to_free = NULL;
1537
1538       uword thread_index = 0;
1539       int index;
1540       const uword nthreads = vlib_num_workers () + 1;
1541       u32 *vec_icmp_bi = NULL;
1542       for (thread_index = 0; thread_index < nthreads; ++thread_index)
1543         {
1544           ip6_full_reass_per_thread_t *rt =
1545             &rm->per_thread_data[thread_index];
1546           clib_spinlock_lock (&rt->lock);
1547
1548           vec_reset_length (pool_indexes_to_free);
1549           pool_foreach_index (index, rt->pool)  {
1550                                 reass = pool_elt_at_index (rt->pool, index);
1551                                 if (now > reass->last_heard + rm->timeout)
1552                                   {
1553                                     vec_add1 (pool_indexes_to_free, index);
1554                                   }
1555                               }
1556           int *i;
1557           vec_foreach (i, pool_indexes_to_free)
1558           {
1559             ip6_full_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1560             u32 icmp_bi = ~0;
1561             ip6_full_reass_on_timeout (vm, node, reass, &icmp_bi);
1562             if (~0 != icmp_bi)
1563               vec_add1 (vec_icmp_bi, icmp_bi);
1564
1565             ip6_full_reass_free (rm, rt, reass);
1566           }
1567
1568           clib_spinlock_unlock (&rt->lock);
1569         }
1570
1571       while (vec_len (vec_icmp_bi) > 0)
1572         {
1573           vlib_frame_t *f =
1574             vlib_get_frame_to_node (vm, rm->ip6_icmp_error_idx);
1575           u32 *to_next = vlib_frame_vector_args (f);
1576           u32 n_left_to_next = VLIB_FRAME_SIZE - f->n_vectors;
1577           int trace_frame = 0;
1578           while (vec_len (vec_icmp_bi) > 0 && n_left_to_next > 0)
1579             {
1580               u32 bi = vec_pop (vec_icmp_bi);
1581               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1582               if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1583                 trace_frame = 1;
1584               b->error = node->errors[IP6_ERROR_REASS_TIMEOUT];
1585               to_next[0] = bi;
1586               ++f->n_vectors;
1587               to_next += 1;
1588               n_left_to_next -= 1;
1589             }
1590           f->frame_flags |= (trace_frame * VLIB_FRAME_TRACE);
1591           vlib_put_frame_to_node (vm, rm->ip6_icmp_error_idx, f);
1592         }
1593
1594       vec_free (pool_indexes_to_free);
1595       vec_free (vec_icmp_bi);
1596       if (event_data)
1597         {
1598           _vec_len (event_data) = 0;
1599         }
1600     }
1601
1602   return 0;
1603 }
1604
1605 VLIB_REGISTER_NODE (ip6_full_reass_expire_node) = {
1606     .function = ip6_full_reass_walk_expired,
1607     .format_trace = format_ip6_full_reass_trace,
1608     .type = VLIB_NODE_TYPE_PROCESS,
1609     .name = "ip6-full-reassembly-expire-walk",
1610
1611     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1612     .error_strings = ip6_full_reassembly_error_strings,
1613
1614 };
1615
1616 static u8 *
1617 format_ip6_full_reass_key (u8 * s, va_list * args)
1618 {
1619   ip6_full_reass_key_t *key = va_arg (*args, ip6_full_reass_key_t *);
1620   s = format (s, "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1621               key->xx_id, format_ip6_address, &key->src, format_ip6_address,
1622               &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1623   return s;
1624 }
1625
1626 static u8 *
1627 format_ip6_full_reass (u8 * s, va_list * args)
1628 {
1629   vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1630   ip6_full_reass_t *reass = va_arg (*args, ip6_full_reass_t *);
1631
1632   s = format (s, "ID: %lu, key: %U\n  first_bi: %u, data_len: %u, "
1633               "last_packet_octet: %u, trace_op_counter: %u\n",
1634               reass->id, format_ip6_full_reass_key, &reass->key,
1635               reass->first_bi, reass->data_len, reass->last_packet_octet,
1636               reass->trace_op_counter);
1637   u32 bi = reass->first_bi;
1638   u32 counter = 0;
1639   while (~0 != bi)
1640     {
1641       vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1642       vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1643       s = format (s, "  #%03u: range: [%u, %u], bi: %u, off: %d, len: %u, "
1644                   "fragment[%u, %u]\n",
1645                   counter, vnb->ip.reass.range_first,
1646                   vnb->ip.reass.range_last, bi,
1647                   ip6_full_reass_buffer_get_data_offset (b),
1648                   ip6_full_reass_buffer_get_data_len (b),
1649                   vnb->ip.reass.fragment_first, vnb->ip.reass.fragment_last);
1650       if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1651         {
1652           bi = b->next_buffer;
1653         }
1654       else
1655         {
1656           bi = ~0;
1657         }
1658     }
1659   return s;
1660 }
1661
1662 static clib_error_t *
1663 show_ip6_full_reass (vlib_main_t * vm, unformat_input_t * input,
1664                      CLIB_UNUSED (vlib_cli_command_t * lmd))
1665 {
1666   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1667
1668   vlib_cli_output (vm, "---------------------");
1669   vlib_cli_output (vm, "IP6 reassembly status");
1670   vlib_cli_output (vm, "---------------------");
1671   bool details = false;
1672   if (unformat (input, "details"))
1673     {
1674       details = true;
1675     }
1676
1677   u32 sum_reass_n = 0;
1678   u64 sum_buffers_n = 0;
1679   ip6_full_reass_t *reass;
1680   uword thread_index;
1681   const uword nthreads = vlib_num_workers () + 1;
1682   for (thread_index = 0; thread_index < nthreads; ++thread_index)
1683     {
1684       ip6_full_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1685       clib_spinlock_lock (&rt->lock);
1686       if (details)
1687         {
1688           pool_foreach (reass, rt->pool) {
1689             vlib_cli_output (vm, "%U", format_ip6_full_reass, vm, reass);
1690           }
1691         }
1692       sum_reass_n += rt->reass_n;
1693       clib_spinlock_unlock (&rt->lock);
1694     }
1695   vlib_cli_output (vm, "---------------------");
1696   vlib_cli_output (vm, "Current IP6 reassemblies count: %lu\n",
1697                    (long unsigned) sum_reass_n);
1698   vlib_cli_output (vm,
1699                    "Maximum configured concurrent full IP6 reassemblies per worker-thread: %lu\n",
1700                    (long unsigned) rm->max_reass_n);
1701   vlib_cli_output (vm,
1702                    "Maximum configured amount of fragments "
1703                    "per full IP6 reassembly: %lu\n",
1704                    (long unsigned) rm->max_reass_len);
1705   vlib_cli_output (vm,
1706                    "Maximum configured full IP6 reassembly timeout: %lums\n",
1707                    (long unsigned) rm->timeout_ms);
1708   vlib_cli_output (vm,
1709                    "Maximum configured full IP6 reassembly expire walk interval: %lums\n",
1710                    (long unsigned) rm->expire_walk_interval_ms);
1711   vlib_cli_output (vm, "Buffers in use: %lu\n",
1712                    (long unsigned) sum_buffers_n);
1713   return 0;
1714 }
1715
1716 VLIB_CLI_COMMAND (show_ip6_full_reassembly_cmd, static) = {
1717     .path = "show ip6-full-reassembly",
1718     .short_help = "show ip6-full-reassembly [details]",
1719     .function = show_ip6_full_reass,
1720 };
1721
1722 #ifndef CLIB_MARCH_VARIANT
1723 vnet_api_error_t
1724 ip6_full_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1725 {
1726   return vnet_feature_enable_disable ("ip6-unicast",
1727                                       "ip6-full-reassembly-feature",
1728                                       sw_if_index, enable_disable, 0, 0);
1729 }
1730 #endif /* CLIB_MARCH_VARIANT */
1731
1732 #define foreach_ip6_full_reassembly_handoff_error                       \
1733 _(CONGESTION_DROP, "congestion drop")
1734
1735
1736 typedef enum
1737 {
1738 #define _(sym,str) IP6_FULL_REASSEMBLY_HANDOFF_ERROR_##sym,
1739   foreach_ip6_full_reassembly_handoff_error
1740 #undef _
1741     IP6_FULL_REASSEMBLY_HANDOFF_N_ERROR,
1742 } ip6_full_reassembly_handoff_error_t;
1743
1744 static char *ip6_full_reassembly_handoff_error_strings[] = {
1745 #define _(sym,string) string,
1746   foreach_ip6_full_reassembly_handoff_error
1747 #undef _
1748 };
1749
1750 typedef struct
1751 {
1752   u32 next_worker_index;
1753 } ip6_full_reassembly_handoff_trace_t;
1754
1755 static u8 *
1756 format_ip6_full_reassembly_handoff_trace (u8 * s, va_list * args)
1757 {
1758   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1759   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1760   ip6_full_reassembly_handoff_trace_t *t =
1761     va_arg (*args, ip6_full_reassembly_handoff_trace_t *);
1762
1763   s =
1764     format (s, "ip6-full-reassembly-handoff: next-worker %d",
1765             t->next_worker_index);
1766
1767   return s;
1768 }
1769
1770 always_inline uword
1771 ip6_full_reassembly_handoff_inline (vlib_main_t * vm,
1772                                     vlib_node_runtime_t * node,
1773                                     vlib_frame_t * frame, bool is_feature)
1774 {
1775   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1776
1777   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1778   u32 n_enq, n_left_from, *from;
1779   u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1780   u32 fq_index;
1781
1782   from = vlib_frame_vector_args (frame);
1783   n_left_from = frame->n_vectors;
1784   vlib_get_buffers (vm, from, bufs, n_left_from);
1785
1786   b = bufs;
1787   ti = thread_indices;
1788
1789   fq_index = (is_feature) ? rm->fq_feature_index : rm->fq_index;
1790
1791   while (n_left_from > 0)
1792     {
1793       ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1794
1795       if (PREDICT_FALSE
1796           ((node->flags & VLIB_NODE_FLAG_TRACE)
1797            && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1798         {
1799           ip6_full_reassembly_handoff_trace_t *t =
1800             vlib_add_trace (vm, node, b[0], sizeof (*t));
1801           t->next_worker_index = ti[0];
1802         }
1803
1804       n_left_from -= 1;
1805       ti += 1;
1806       b += 1;
1807     }
1808   n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1809                                          thread_indices, frame->n_vectors, 1);
1810
1811   if (n_enq < frame->n_vectors)
1812     vlib_node_increment_counter (vm, node->node_index,
1813                                  IP6_FULL_REASSEMBLY_HANDOFF_ERROR_CONGESTION_DROP,
1814                                  frame->n_vectors - n_enq);
1815   return frame->n_vectors;
1816 }
1817
1818 VLIB_NODE_FN (ip6_full_reassembly_handoff_node) (vlib_main_t * vm,
1819                                                  vlib_node_runtime_t * node,
1820                                                  vlib_frame_t * frame)
1821 {
1822   return ip6_full_reassembly_handoff_inline (vm, node, frame,
1823                                              false /* is_feature */ );
1824 }
1825
1826 VLIB_REGISTER_NODE (ip6_full_reassembly_handoff_node) = {
1827   .name = "ip6-full-reassembly-handoff",
1828   .vector_size = sizeof (u32),
1829   .n_errors = ARRAY_LEN(ip6_full_reassembly_handoff_error_strings),
1830   .error_strings = ip6_full_reassembly_handoff_error_strings,
1831   .format_trace = format_ip6_full_reassembly_handoff_trace,
1832
1833   .n_next_nodes = 1,
1834
1835   .next_nodes = {
1836     [0] = "error-drop",
1837   },
1838 };
1839
1840
1841 VLIB_NODE_FN (ip6_full_reassembly_feature_handoff_node) (vlib_main_t * vm,
1842                                vlib_node_runtime_t * node, vlib_frame_t * frame)
1843 {
1844   return ip6_full_reassembly_handoff_inline (vm, node, frame, true /* is_feature */ );
1845 }
1846
1847
1848 VLIB_REGISTER_NODE (ip6_full_reassembly_feature_handoff_node) = {
1849   .name = "ip6-full-reass-feature-hoff",
1850   .vector_size = sizeof (u32),
1851   .n_errors = ARRAY_LEN(ip6_full_reassembly_handoff_error_strings),
1852   .error_strings = ip6_full_reassembly_handoff_error_strings,
1853   .format_trace = format_ip6_full_reassembly_handoff_trace,
1854
1855   .n_next_nodes = 1,
1856
1857   .next_nodes = {
1858     [0] = "error-drop",
1859   },
1860 };
1861
1862 #ifndef CLIB_MARCH_VARIANT
1863 int
1864 ip6_full_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
1865 {
1866   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1867   vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
1868   if (is_enable)
1869     {
1870       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1871         {
1872           ++rm->feature_use_refcount_per_intf[sw_if_index];
1873           return vnet_feature_enable_disable ("ip6-unicast",
1874                                               "ip6-full-reassembly-feature",
1875                                               sw_if_index, 1, 0, 0);
1876         }
1877       ++rm->feature_use_refcount_per_intf[sw_if_index];
1878     }
1879   else
1880     {
1881       --rm->feature_use_refcount_per_intf[sw_if_index];
1882       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1883         return vnet_feature_enable_disable ("ip6-unicast",
1884                                             "ip6-full-reassembly-feature",
1885                                             sw_if_index, 0, 0, 0);
1886     }
1887   return -1;
1888 }
1889 #endif
1890
1891 /*
1892  * fd.io coding-style-patch-verification: ON
1893  *
1894  * Local Variables:
1895  * eval: (c-set-style "gnu")
1896  * End:
1897  */