src/vnet/ip/reass/ip6_full_reass.c

   1 /*
   2  * Copyright (c) 2017 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 /**
  17  * @file
  18  * @brief IPv6 Full Reassembly.
  19  *
  20  * This file contains the source code for IPv6 full reassembly.
  21  */
  22
  23 #include <vppinfra/vec.h>
  24 #include <vnet/vnet.h>
  25 #include <vnet/ip/ip.h>
  26 #include <vppinfra/bihash_48_8.h>
  27 #include <vnet/ip/reass/ip6_full_reass.h>
  28 #include <vnet/ip/ip6_inlines.h>
  29
  30 #define MSEC_PER_SEC 1000
  31 #define IP6_FULL_REASS_TIMEOUT_DEFAULT_MS 100
  32 #define IP6_FULL_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS 10000    // 10 seconds default
  33 #define IP6_FULL_REASS_MAX_REASSEMBLIES_DEFAULT 1024
  34 #define IP6_FULL_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT 3
  35 #define IP6_FULL_REASS_HT_LOAD_FACTOR (0.75)
  36
  37 typedef enum
  38 {
  39   IP6_FULL_REASS_RC_OK,
  40   IP6_FULL_REASS_RC_INTERNAL_ERROR,
  41   IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS,
  42   IP6_FULL_REASS_RC_NO_BUF,
  43   IP6_FULL_REASS_RC_HANDOFF,
  44   IP6_FULL_REASS_RC_INVALID_FRAG_LEN,
  45   IP6_FULL_REASS_RC_OVERLAP,
  46 } ip6_full_reass_rc_t;
  47
  48 typedef struct
  49 {
  50   union
  51   {
  52     struct
  53     {
  54       ip6_address_t src;
  55       ip6_address_t dst;
  56       u32 xx_id;
  57       u32 frag_id;
  58       u8 unused[7];
  59       u8 proto;
  60     };
  61     u64 as_u64[6];
  62   };
  63 } ip6_full_reass_key_t;
  64
  65 typedef union
  66 {
  67   struct
  68   {
  69     u32 reass_index;
  70     u32 memory_owner_thread_index;
  71   };
  72   u64 as_u64;
  73 } ip6_full_reass_val_t;
  74
  75 typedef union
  76 {
  77   struct
  78   {
  79     ip6_full_reass_key_t k;
  80     ip6_full_reass_val_t v;
  81   };
  82   clib_bihash_kv_48_8_t kv;
  83 } ip6_full_reass_kv_t;
  84
  85
  86 always_inline u32
  87 ip6_full_reass_buffer_get_data_offset (vlib_buffer_t * b)
  88 {
  89   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  90   return vnb->ip.reass.range_first - vnb->ip.reass.fragment_first;
  91 }
  92
  93 always_inline u16
  94 ip6_full_reass_buffer_get_data_len (vlib_buffer_t * b)
  95 {
  96   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
  97   return clib_min (vnb->ip.reass.range_last, vnb->ip.reass.fragment_last) -
  98     (vnb->ip.reass.fragment_first +
  99      ip6_full_reass_buffer_get_data_offset (b)) + 1;
 100 }
 101
 102 typedef struct
 103 {
 104   // hash table key
 105   ip6_full_reass_key_t key;
 106   // time when last packet was received
 107   f64 last_heard;
 108   // internal id of this reassembly
 109   u64 id;
 110   // buffer index of first buffer in this reassembly context
 111   u32 first_bi;
 112   // last octet of packet, ~0 until fragment without more_fragments arrives
 113   u32 last_packet_octet;
 114   // length of data collected so far
 115   u32 data_len;
 116   // trace operation counter
 117   u32 trace_op_counter;
 118   // next index - used by custom apps (~0 if not set)
 119   u32 next_index;
 120   // error next index - used by custom apps (~0 if not set)
 121   u32 error_next_index;
 122   // minimum fragment length for this reassembly - used to estimate MTU
 123   u16 min_fragment_length;
 124   // number of fragments for this reassembly
 125   u32 fragments_n;
 126   // thread owning memory for this context (whose pool contains this ctx)
 127   u32 memory_owner_thread_index;
 128   // thread which received fragment with offset 0 and which sends out the
 129   // completed reassembly
 130   u32 sendout_thread_index;
 131 } ip6_full_reass_t;
 132
 133 typedef struct
 134 {
 135   ip6_full_reass_t *pool;
 136   u32 reass_n;
 137   u32 id_counter;
 138   clib_spinlock_t lock;
 139 } ip6_full_reass_per_thread_t;
 140
 141 typedef struct
 142 {
 143   // IPv6 config
 144   u32 timeout_ms;
 145   f64 timeout;
 146   u32 expire_walk_interval_ms;
 147   // maximum number of fragments in one reassembly
 148   u32 max_reass_len;
 149   // maximum number of reassemblies
 150   u32 max_reass_n;
 151
 152   // IPv6 runtime
 153   clib_bihash_48_8_t hash;
 154
 155   // per-thread data
 156   ip6_full_reass_per_thread_t *per_thread_data;
 157
 158   // convenience
 159   vlib_main_t *vlib_main;
 160
 161   u32 ip6_icmp_error_idx;
 162   u32 ip6_full_reass_expire_node_idx;
 163
 164   /** Worker handoff */
 165   u32 fq_index;
 166   u32 fq_local_index;
 167   u32 fq_feature_index;
 168
 169   // reference count for enabling/disabling feature - per interface
 170   u32 *feature_use_refcount_per_intf;
 171
 172   // whether local fragmented packets are reassembled or not
 173   int is_local_reass_enabled;
 174 } ip6_full_reass_main_t;
 175
 176 extern ip6_full_reass_main_t ip6_full_reass_main;
 177
 178 #ifndef CLIB_MARCH_VARIANT
 179 ip6_full_reass_main_t ip6_full_reass_main;
 180 #endif /* CLIB_MARCH_VARIANT */
 181
 182 typedef enum
 183 {
 184   IP6_FULL_REASSEMBLY_NEXT_INPUT,
 185   IP6_FULL_REASSEMBLY_NEXT_DROP,
 186   IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR,
 187   IP6_FULL_REASSEMBLY_NEXT_HANDOFF,
 188   IP6_FULL_REASSEMBLY_N_NEXT,
 189 } ip6_full_reass_next_t;
 190
 191 typedef enum
 192 {
 193   RANGE_NEW,
 194   RANGE_OVERLAP,
 195   ICMP_ERROR_RT_EXCEEDED,
 196   ICMP_ERROR_FL_TOO_BIG,
 197   ICMP_ERROR_FL_NOT_MULT_8,
 198   FINALIZE,
 199   HANDOFF,
 200   PASSTHROUGH,
 201 } ip6_full_reass_trace_operation_e;
 202
 203 typedef struct
 204 {
 205   u16 range_first;
 206   u16 range_last;
 207   u32 range_bi;
 208   i32 data_offset;
 209   u32 data_len;
 210   u32 first_bi;
 211 } ip6_full_reass_range_trace_t;
 212
 213 typedef struct
 214 {
 215   ip6_full_reass_trace_operation_e action;
 216   u32 reass_id;
 217   ip6_full_reass_range_trace_t trace_range;
 218   u32 op_id;
 219   u32 fragment_first;
 220   u32 fragment_last;
 221   u32 total_data_len;
 222   u32 thread_id;
 223   u32 thread_id_to;
 224   bool is_after_handoff;
 225   ip6_header_t ip6_header;
 226   ip6_frag_hdr_t ip6_frag_header;
 227 } ip6_full_reass_trace_t;
 228
 229 static void
 230 ip6_full_reass_trace_details (vlib_main_t * vm, u32 bi,
 231                               ip6_full_reass_range_trace_t * trace)
 232 {
 233   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 234   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 235   trace->range_first = vnb->ip.reass.range_first;
 236   trace->range_last = vnb->ip.reass.range_last;
 237   trace->data_offset = ip6_full_reass_buffer_get_data_offset (b);
 238   trace->data_len = ip6_full_reass_buffer_get_data_len (b);
 239   trace->range_bi = bi;
 240 }
 241
 242 static u8 *
 243 format_ip6_full_reass_range_trace (u8 * s, va_list * args)
 244 {
 245   ip6_full_reass_range_trace_t *trace =
 246     va_arg (*args, ip6_full_reass_range_trace_t *);
 247   s =
 248     format (s, "range: [%u, %u], off %d, len %u, bi %u", trace->range_first,
 249             trace->range_last, trace->data_offset, trace->data_len,
 250             trace->range_bi);
 251   return s;
 252 }
 253
 254 static u8 *
 255 format_ip6_full_reass_trace (u8 * s, va_list * args)
 256 {
 257   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 258   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 259   ip6_full_reass_trace_t *t = va_arg (*args, ip6_full_reass_trace_t *);
 260   u32 indent = 0;
 261   if (~0 != t->reass_id)
 262     {
 263       if (t->is_after_handoff)
 264         {
 265           s =
 266             format (s, "%U\n", format_ip6_header, &t->ip6_header,
 267                     sizeof (t->ip6_header));
 268           s =
 269             format (s, "  %U\n", format_ip6_frag_hdr, &t->ip6_frag_header,
 270                     sizeof (t->ip6_frag_header));
 271           indent = 2;
 272         }
 273       s =
 274         format (s, "%Ureass id: %u, op id: %u, ", format_white_space, indent,
 275                 t->reass_id, t->op_id);
 276       indent = format_get_indent (s);
 277       s = format (s, "first bi: %u, data len: %u, ip/fragment[%u, %u]",
 278                   t->trace_range.first_bi, t->total_data_len,
 279                   t->fragment_first, t->fragment_last);
 280     }
 281   switch (t->action)
 282     {
 283     case RANGE_NEW:
 284       s = format (s, "\n%Unew %U", format_white_space, indent,
 285                   format_ip6_full_reass_range_trace, &t->trace_range);
 286       break;
 287     case RANGE_OVERLAP:
 288       s = format (s, "\n%Uoverlap %U", format_white_space, indent,
 289                   format_ip6_full_reass_range_trace, &t->trace_range);
 290       break;
 291     case ICMP_ERROR_FL_TOO_BIG:
 292       s = format (s, "\n%Uicmp-error - frag_len > 65535 %U",
 293                   format_white_space, indent,
 294                   format_ip6_full_reass_range_trace, &t->trace_range);
 295       break;
 296     case ICMP_ERROR_FL_NOT_MULT_8:
 297       s = format (s, "\n%Uicmp-error - frag_len mod 8 != 0 %U",
 298                   format_white_space, indent,
 299                   format_ip6_full_reass_range_trace, &t->trace_range);
 300       break;
 301     case ICMP_ERROR_RT_EXCEEDED:
 302       s = format (s, "\n%Uicmp-error - reassembly time exceeded",
 303                   format_white_space, indent);
 304       break;
 305     case FINALIZE:
 306       s = format (s, "\n%Ufinalize reassembly", format_white_space, indent);
 307       break;
 308     case HANDOFF:
 309       s =
 310         format (s, "handoff from thread #%u to thread #%u", t->thread_id,
 311                 t->thread_id_to);
 312       break;
 313     case PASSTHROUGH:
 314       s = format (s, "passthrough - not a fragment");
 315       break;
 316     }
 317   return s;
 318 }
 319
 320 static void
 321 ip6_full_reass_add_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
 322                           ip6_full_reass_t * reass, u32 bi,
 323                           ip6_frag_hdr_t * ip6_frag_header,
 324                           ip6_full_reass_trace_operation_e action,
 325                           u32 thread_id_to)
 326 {
 327   vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 328   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
 329   bool is_after_handoff = false;
 330   if (pool_is_free_index
 331       (vm->trace_main.trace_buffer_pool, vlib_buffer_get_trace_index (b)))
 332     {
 333       // this buffer's trace is gone
 334       b->flags &= ~VLIB_BUFFER_IS_TRACED;
 335       return;
 336     }
 337   if (vlib_buffer_get_trace_thread (b) != vm->thread_index)
 338     {
 339       is_after_handoff = true;
 340     }
 341   ip6_full_reass_trace_t *t = vlib_add_trace (vm, node, b, sizeof (t[0]));
 342   t->is_after_handoff = is_after_handoff;
 343   if (t->is_after_handoff)
 344     {
 345       clib_memcpy (&t->ip6_header, vlib_buffer_get_current (b),
 346                    clib_min (sizeof (t->ip6_header), b->current_length));
 347       if (ip6_frag_header)
 348         {
 349           clib_memcpy (&t->ip6_frag_header, ip6_frag_header,
 350                        sizeof (t->ip6_frag_header));
 351         }
 352       else
 353         {
 354           clib_memset (&t->ip6_frag_header, 0, sizeof (t->ip6_frag_header));
 355         }
 356     }
 357   if (reass)
 358     {
 359       t->reass_id = reass->id;
 360       t->op_id = reass->trace_op_counter;
 361       t->trace_range.first_bi = reass->first_bi;
 362       t->total_data_len = reass->data_len;
 363       ++reass->trace_op_counter;
 364     }
 365   else
 366     {
 367       t->reass_id = ~0;
 368     }
 369   t->action = action;
 370   t->thread_id = vm->thread_index;
 371   t->thread_id_to = thread_id_to;
 372   ip6_full_reass_trace_details (vm, bi, &t->trace_range);
 373   t->fragment_first = vnb->ip.reass.fragment_first;
 374   t->fragment_last = vnb->ip.reass.fragment_last;
 375 #if 0
 376   static u8 *s = NULL;
 377   s = format (s, "%U", format_ip6_full_reass_trace, NULL, NULL, t);
 378   printf ("%.*s\n", vec_len (s), s);
 379   fflush (stdout);
 380   vec_reset_length (s);
 381 #endif
 382 }
 383
 384 always_inline void
 385 ip6_full_reass_free_ctx (ip6_full_reass_per_thread_t * rt,
 386                          ip6_full_reass_t * reass)
 387 {
 388   pool_put (rt->pool, reass);
 389   --rt->reass_n;
 390 }
 391
 392 always_inline void
 393 ip6_full_reass_free (ip6_full_reass_main_t * rm,
 394                      ip6_full_reass_per_thread_t * rt,
 395                      ip6_full_reass_t * reass)
 396 {
 397   clib_bihash_kv_48_8_t kv;
 398   kv.key[0] = reass->key.as_u64[0];
 399   kv.key[1] = reass->key.as_u64[1];
 400   kv.key[2] = reass->key.as_u64[2];
 401   kv.key[3] = reass->key.as_u64[3];
 402   kv.key[4] = reass->key.as_u64[4];
 403   kv.key[5] = reass->key.as_u64[5];
 404   clib_bihash_add_del_48_8 (&rm->hash, &kv, 0);
 405   ip6_full_reass_free_ctx (rt, reass);
 406 }
 407
 408 always_inline void
 409 ip6_full_reass_drop_all (vlib_main_t *vm, vlib_node_runtime_t *node,
 410                          ip6_full_reass_t *reass, u32 offending_bi)
 411 {
 412   u32 range_bi = reass->first_bi;
 413   vlib_buffer_t *range_b;
 414   vnet_buffer_opaque_t *range_vnb;
 415   u32 *to_free = NULL;
 416   while (~0 != range_bi)
 417     {
 418       range_b = vlib_get_buffer (vm, range_bi);
 419       range_vnb = vnet_buffer (range_b);
 420       u32 bi = range_bi;
 421       while (~0 != bi)
 422         {
 423           vec_add1 (to_free, bi);
 424           if (bi == offending_bi)
 425             {
 426               offending_bi = ~0;
 427             }
 428           vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 429           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 430             {
 431               bi = b->next_buffer;
 432               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 433             }
 434           else
 435             {
 436               bi = ~0;
 437             }
 438         }
 439       range_bi = range_vnb->ip.reass.next_range_bi;
 440     }
 441   if (~0 != offending_bi)
 442     {
 443       vec_add1 (to_free, offending_bi);
 444     }
 445   /* send to next_error_index */
 446   if (~0 != reass->error_next_index)
 447     {
 448       u32 n_left_to_next, *to_next, next_index;
 449
 450       next_index = reass->error_next_index;
 451       u32 bi = ~0;
 452
 453       while (vec_len (to_free) > 0)
 454         {
 455           vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 456
 457           while (vec_len (to_free) > 0 && n_left_to_next > 0)
 458             {
 459               bi = vec_pop (to_free);
 460
 461               if (~0 != bi)
 462                 {
 463                   to_next[0] = bi;
 464                   to_next += 1;
 465                   n_left_to_next -= 1;
 466                 }
 467             }
 468           vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 469         }
 470     }
 471   else
 472     {
 473       vlib_buffer_free (vm, to_free, vec_len (to_free));
 474     }
 475   vec_free (to_free);
 476 }
 477
 478 always_inline void
 479 ip6_full_reass_on_timeout (vlib_main_t * vm, vlib_node_runtime_t * node,
 480                            ip6_full_reass_t * reass, u32 * icmp_bi)
 481 {
 482   if (~0 == reass->first_bi)
 483     {
 484       return;
 485     }
 486   if (~0 == reass->next_index)  // custom apps don't want icmp
 487     {
 488       vlib_buffer_t *b = vlib_get_buffer (vm, reass->first_bi);
 489       if (0 == vnet_buffer (b)->ip.reass.fragment_first)
 490         {
 491           *icmp_bi = reass->first_bi;
 492           if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
 493             {
 494               ip6_full_reass_add_trace (vm, node, reass, reass->first_bi, NULL,
 495                                         ICMP_ERROR_RT_EXCEEDED, ~0);
 496             }
 497           // fragment with offset zero received - send icmp message back
 498           if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 499             {
 500               // separate first buffer from chain and steer it towards icmp node
 501               b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 502               reass->first_bi = b->next_buffer;
 503             }
 504           else
 505             {
 506               reass->first_bi = vnet_buffer (b)->ip.reass.next_range_bi;
 507             }
 508           icmp6_error_set_vnet_buffer (b, ICMP6_time_exceeded,
 509                                        ICMP6_time_exceeded_fragment_reassembly_time_exceeded,
 510                                        0);
 511         }
 512     }
 513   ip6_full_reass_drop_all (vm, node, reass, ~0);
 514 }
 515
 516 always_inline ip6_full_reass_t *
 517 ip6_full_reass_find_or_create (vlib_main_t *vm, vlib_node_runtime_t *node,
 518                                ip6_full_reass_main_t *rm,
 519                                ip6_full_reass_per_thread_t *rt,
 520                                ip6_full_reass_kv_t *kv, u32 *icmp_bi,
 521                                u8 *do_handoff, int skip_bihash)
 522 {
 523   ip6_full_reass_t *reass;
 524   f64 now;
 525
 526 again:
 527
 528   reass = NULL;
 529   now = vlib_time_now (vm);
 530
 531   if (!skip_bihash && !clib_bihash_search_48_8 (&rm->hash, &kv->kv, &kv->kv))
 532     {
 533       if (vm->thread_index != kv->v.memory_owner_thread_index)
 534         {
 535           *do_handoff = 1;
 536           return NULL;
 537         }
 538
 539       reass =
 540         pool_elt_at_index (rm->per_thread_data
 541                            [kv->v.memory_owner_thread_index].pool,
 542                            kv->v.reass_index);
 543
 544       if (now > reass->last_heard + rm->timeout)
 545         {
 546           ip6_full_reass_on_timeout (vm, node, reass, icmp_bi);
 547           ip6_full_reass_free (rm, rt, reass);
 548           reass = NULL;
 549         }
 550     }
 551
 552   if (reass)
 553     {
 554       reass->last_heard = now;
 555       return reass;
 556     }
 557
 558   if (rt->reass_n >= rm->max_reass_n)
 559     {
 560       reass = NULL;
 561       return reass;
 562     }
 563   else
 564     {
 565       pool_get (rt->pool, reass);
 566       clib_memset (reass, 0, sizeof (*reass));
 567       reass->id = ((u64) vm->thread_index * 1000000000) + rt->id_counter;
 568       ++rt->id_counter;
 569       reass->first_bi = ~0;
 570       reass->last_packet_octet = ~0;
 571       reass->data_len = 0;
 572       reass->next_index = ~0;
 573       reass->error_next_index = ~0;
 574       reass->memory_owner_thread_index = vm->thread_index;
 575       ++rt->reass_n;
 576     }
 577
 578   kv->v.reass_index = (reass - rt->pool);
 579   kv->v.memory_owner_thread_index = vm->thread_index;
 580   reass->last_heard = now;
 581
 582   if (!skip_bihash)
 583     {
 584       reass->key.as_u64[0] = kv->kv.key[0];
 585       reass->key.as_u64[1] = kv->kv.key[1];
 586       reass->key.as_u64[2] = kv->kv.key[2];
 587       reass->key.as_u64[3] = kv->kv.key[3];
 588       reass->key.as_u64[4] = kv->kv.key[4];
 589       reass->key.as_u64[5] = kv->kv.key[5];
 590
 591       int rv = clib_bihash_add_del_48_8 (&rm->hash, &kv->kv, 2);
 592       if (rv)
 593         {
 594           ip6_full_reass_free (rm, rt, reass);
 595           reass = NULL;
 596           // if other worker created a context already work with the other copy
 597           if (-2 == rv)
 598             goto again;
 599         }
 600     }
 601   else
 602     {
 603       reass->key.as_u64[0] = ~0;
 604       reass->key.as_u64[1] = ~0;
 605       reass->key.as_u64[2] = ~0;
 606       reass->key.as_u64[3] = ~0;
 607       reass->key.as_u64[4] = ~0;
 608       reass->key.as_u64[5] = ~0;
 609     }
 610
 611   return reass;
 612 }
 613
 614 always_inline ip6_full_reass_rc_t
 615 ip6_full_reass_finalize (vlib_main_t * vm, vlib_node_runtime_t * node,
 616                          ip6_full_reass_main_t * rm,
 617                          ip6_full_reass_per_thread_t * rt,
 618                          ip6_full_reass_t * reass, u32 * bi0, u32 * next0,
 619                          u32 * error0, bool is_custom_app)
 620 {
 621   *bi0 = reass->first_bi;
 622   *error0 = IP6_ERROR_NONE;
 623   ip6_frag_hdr_t *frag_hdr;
 624   vlib_buffer_t *last_b = NULL;
 625   u32 sub_chain_bi = reass->first_bi;
 626   u32 total_length = 0;
 627   u32 buf_cnt = 0;
 628   u32 dropped_cnt = 0;
 629   u32 *vec_drop_compress = NULL;
 630   ip6_full_reass_rc_t rv = IP6_FULL_REASS_RC_OK;
 631   do
 632     {
 633       u32 tmp_bi = sub_chain_bi;
 634       vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi);
 635       vnet_buffer_opaque_t *vnb = vnet_buffer (tmp);
 636       if (!(vnb->ip.reass.range_first >= vnb->ip.reass.fragment_first) &&
 637           !(vnb->ip.reass.range_last > vnb->ip.reass.fragment_first))
 638         {
 639           rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 640           goto free_buffers_and_return;
 641         }
 642
 643       u32 data_len = ip6_full_reass_buffer_get_data_len (tmp);
 644       u32 trim_front = vnet_buffer (tmp)->ip.reass.ip6_frag_hdr_offset +
 645         sizeof (*frag_hdr) + ip6_full_reass_buffer_get_data_offset (tmp);
 646       u32 trim_end =
 647         vlib_buffer_length_in_chain (vm, tmp) - trim_front - data_len;
 648       if (tmp_bi == reass->first_bi)
 649         {
 650           /* first buffer - keep ip6 header */
 651           if (0 != ip6_full_reass_buffer_get_data_offset (tmp))
 652             {
 653               rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 654               goto free_buffers_and_return;
 655             }
 656           trim_front = 0;
 657           trim_end = vlib_buffer_length_in_chain (vm, tmp) - data_len -
 658             (vnet_buffer (tmp)->ip.reass.ip6_frag_hdr_offset +
 659              sizeof (*frag_hdr));
 660           if (!(vlib_buffer_length_in_chain (vm, tmp) - trim_end > 0))
 661             {
 662               rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 663               goto free_buffers_and_return;
 664             }
 665         }
 666       u32 keep_data =
 667         vlib_buffer_length_in_chain (vm, tmp) - trim_front - trim_end;
 668       while (1)
 669         {
 670           ++buf_cnt;
 671           if (trim_front)
 672             {
 673               if (trim_front > tmp->current_length)
 674                 {
 675                   /* drop whole buffer */
 676                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 677                     {
 678                       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 679                       goto free_buffers_and_return;
 680                     }
 681                   trim_front -= tmp->current_length;
 682                   vec_add1 (vec_drop_compress, tmp_bi);
 683                   tmp->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 684                   tmp_bi = tmp->next_buffer;
 685                   tmp = vlib_get_buffer (vm, tmp_bi);
 686                   continue;
 687                 }
 688               else
 689                 {
 690                   vlib_buffer_advance (tmp, trim_front);
 691                   trim_front = 0;
 692                 }
 693             }
 694           if (keep_data)
 695             {
 696               if (last_b)
 697                 {
 698                   last_b->flags |= VLIB_BUFFER_NEXT_PRESENT;
 699                   last_b->next_buffer = tmp_bi;
 700                 }
 701               last_b = tmp;
 702               if (keep_data <= tmp->current_length)
 703                 {
 704                   tmp->current_length = keep_data;
 705                   keep_data = 0;
 706                 }
 707               else
 708                 {
 709                   keep_data -= tmp->current_length;
 710                   if (!(tmp->flags & VLIB_BUFFER_NEXT_PRESENT))
 711                     {
 712                       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 713                       goto free_buffers_and_return;
 714                     }
 715                 }
 716               total_length += tmp->current_length;
 717             }
 718           else
 719             {
 720               if (reass->first_bi == tmp_bi)
 721                 {
 722                   rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 723                   goto free_buffers_and_return;
 724                 }
 725               vec_add1 (vec_drop_compress, tmp_bi);
 726               ++dropped_cnt;
 727             }
 728           if (tmp->flags & VLIB_BUFFER_NEXT_PRESENT)
 729             {
 730               tmp_bi = tmp->next_buffer;
 731               tmp = vlib_get_buffer (vm, tmp->next_buffer);
 732             }
 733           else
 734             {
 735               break;
 736             }
 737         }
 738       sub_chain_bi =
 739         vnet_buffer (vlib_get_buffer (vm, sub_chain_bi))->ip.
 740         reass.next_range_bi;
 741     }
 742   while (~0 != sub_chain_bi);
 743
 744   if (!last_b)
 745     {
 746       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 747       goto free_buffers_and_return;
 748     }
 749   last_b->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 750   vlib_buffer_t *first_b = vlib_get_buffer (vm, reass->first_bi);
 751   if (total_length < first_b->current_length)
 752     {
 753       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 754       goto free_buffers_and_return;
 755     }
 756   total_length -= first_b->current_length;
 757   first_b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 758   first_b->total_length_not_including_first_buffer = total_length;
 759   // drop fragment header
 760   vnet_buffer_opaque_t *first_b_vnb = vnet_buffer (first_b);
 761   ip6_header_t *ip = vlib_buffer_get_current (first_b);
 762   u16 ip6_frag_hdr_offset = first_b_vnb->ip.reass.ip6_frag_hdr_offset;
 763   ip6_ext_hdr_chain_t hdr_chain;
 764   ip6_ext_header_t *prev_hdr = 0;
 765   int res = ip6_ext_header_walk (first_b, ip, IP_PROTOCOL_IPV6_FRAGMENTATION,
 766                                  &hdr_chain);
 767   if (res < 0 ||
 768       (hdr_chain.eh[res].protocol != IP_PROTOCOL_IPV6_FRAGMENTATION))
 769     {
 770       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 771       goto free_buffers_and_return;
 772     }
 773   frag_hdr = ip6_ext_next_header_offset (ip, hdr_chain.eh[res].offset);
 774   if (res > 0)
 775     {
 776       prev_hdr = ip6_ext_next_header_offset (ip, hdr_chain.eh[res - 1].offset);
 777       prev_hdr->next_hdr = frag_hdr->next_hdr;
 778     }
 779   else
 780     {
 781       ip->protocol = frag_hdr->next_hdr;
 782     }
 783   if (hdr_chain.eh[res].offset != ip6_frag_hdr_offset)
 784     {
 785       rv = IP6_FULL_REASS_RC_INTERNAL_ERROR;
 786       goto free_buffers_and_return;
 787     }
 788   memmove (frag_hdr, (u8 *) frag_hdr + sizeof (*frag_hdr),
 789            first_b->current_length - ip6_frag_hdr_offset -
 790            sizeof (ip6_frag_hdr_t));
 791   first_b->current_length -= sizeof (*frag_hdr);
 792   ip->payload_length =
 793     clib_host_to_net_u16 (total_length + first_b->current_length -
 794                           sizeof (*ip));
 795   if (!vlib_buffer_chain_linearize (vm, first_b))
 796     {
 797       rv = IP6_FULL_REASS_RC_NO_BUF;
 798       goto free_buffers_and_return;
 799     }
 800   first_b->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
 801   if (PREDICT_FALSE (first_b->flags & VLIB_BUFFER_IS_TRACED))
 802     {
 803       ip6_full_reass_add_trace (vm, node, reass, reass->first_bi, NULL,
 804                                 FINALIZE, ~0);
 805 #if 0
 806       // following code does a hexdump of packet fragments to stdout ...
 807       do
 808         {
 809           u32 bi = reass->first_bi;
 810           u8 *s = NULL;
 811           while (~0 != bi)
 812             {
 813               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
 814               s = format (s, "%u: %U\n", bi, format_hexdump,
 815                           vlib_buffer_get_current (b), b->current_length);
 816               if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
 817                 {
 818                   bi = b->next_buffer;
 819                 }
 820               else
 821                 {
 822                   break;
 823                 }
 824             }
 825           printf ("%.*s\n", vec_len (s), s);
 826           fflush (stdout);
 827           vec_free (s);
 828         }
 829       while (0);
 830 #endif
 831     }
 832   if (!is_custom_app)
 833     {
 834       *next0 = IP6_FULL_REASSEMBLY_NEXT_INPUT;
 835     }
 836   else
 837     {
 838       *next0 = reass->next_index;
 839     }
 840   vnet_buffer (first_b)->ip.reass.estimated_mtu = reass->min_fragment_length;
 841   ip6_full_reass_free (rm, rt, reass);
 842   reass = NULL;
 843 free_buffers_and_return:
 844   vlib_buffer_free (vm, vec_drop_compress, vec_len (vec_drop_compress));
 845   vec_free (vec_drop_compress);
 846   return rv;
 847 }
 848
 849 always_inline void
 850 ip6_full_reass_insert_range_in_chain (vlib_main_t * vm,
 851                                       ip6_full_reass_t * reass,
 852                                       u32 prev_range_bi, u32 new_next_bi)
 853 {
 854
 855   vlib_buffer_t *new_next_b = vlib_get_buffer (vm, new_next_bi);
 856   vnet_buffer_opaque_t *new_next_vnb = vnet_buffer (new_next_b);
 857   if (~0 != prev_range_bi)
 858     {
 859       vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_range_bi);
 860       vnet_buffer_opaque_t *prev_vnb = vnet_buffer (prev_b);
 861       new_next_vnb->ip.reass.next_range_bi = prev_vnb->ip.reass.next_range_bi;
 862       prev_vnb->ip.reass.next_range_bi = new_next_bi;
 863     }
 864   else
 865     {
 866       if (~0 != reass->first_bi)
 867         {
 868           new_next_vnb->ip.reass.next_range_bi = reass->first_bi;
 869         }
 870       reass->first_bi = new_next_bi;
 871     }
 872   reass->data_len += ip6_full_reass_buffer_get_data_len (new_next_b);
 873 }
 874
 875 always_inline ip6_full_reass_rc_t
 876 ip6_full_reass_update (vlib_main_t *vm, vlib_node_runtime_t *node,
 877                        ip6_full_reass_main_t *rm,
 878                        ip6_full_reass_per_thread_t *rt,
 879                        ip6_full_reass_t *reass, u32 *bi0, u32 *next0,
 880                        u32 *error0, ip6_frag_hdr_t *frag_hdr,
 881                        bool is_custom_app, u32 *handoff_thread_idx,
 882                        int skip_bihash)
 883 {
 884   int consumed = 0;
 885   vlib_buffer_t *fb = vlib_get_buffer (vm, *bi0);
 886   vnet_buffer_opaque_t *fvnb = vnet_buffer (fb);
 887   if (is_custom_app)
 888     {
 889       reass->next_index = fvnb->ip.reass.next_index;    // store next_index before it's overwritten
 890       reass->error_next_index = fvnb->ip.reass.error_next_index;        // store error_next_index before it is overwritten
 891     }
 892
 893   fvnb->ip.reass.ip6_frag_hdr_offset =
 894     (u8 *) frag_hdr - (u8 *) vlib_buffer_get_current (fb);
 895   ip6_header_t *fip = vlib_buffer_get_current (fb);
 896   if (fb->current_length < sizeof (*fip) ||
 897       fvnb->ip.reass.ip6_frag_hdr_offset == 0 ||
 898       fvnb->ip.reass.ip6_frag_hdr_offset >= fb->current_length)
 899     {
 900       return IP6_FULL_REASS_RC_INTERNAL_ERROR;
 901     }
 902
 903   u32 fragment_first = fvnb->ip.reass.fragment_first =
 904     ip6_frag_hdr_offset_bytes (frag_hdr);
 905   u32 fragment_length =
 906     vlib_buffer_length_in_chain (vm, fb) -
 907     (fvnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
 908   if (0 == fragment_length)
 909     {
 910       return IP6_FULL_REASS_RC_INVALID_FRAG_LEN;
 911     }
 912   u32 fragment_last = fvnb->ip.reass.fragment_last =
 913     fragment_first + fragment_length - 1;
 914   int more_fragments = ip6_frag_hdr_more (frag_hdr);
 915   u32 candidate_range_bi = reass->first_bi;
 916   u32 prev_range_bi = ~0;
 917   fvnb->ip.reass.range_first = fragment_first;
 918   fvnb->ip.reass.range_last = fragment_last;
 919   fvnb->ip.reass.next_range_bi = ~0;
 920   if (!more_fragments)
 921     {
 922       reass->last_packet_octet = fragment_last;
 923     }
 924   if (~0 == reass->first_bi)
 925     {
 926       // starting a new reassembly
 927       ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi, *bi0);
 928       reass->min_fragment_length = clib_net_to_host_u16 (fip->payload_length);
 929       consumed = 1;
 930       reass->fragments_n = 1;
 931       goto check_if_done_maybe;
 932     }
 933   reass->min_fragment_length =
 934     clib_min (clib_net_to_host_u16 (fip->payload_length),
 935               fvnb->ip.reass.estimated_mtu);
 936   while (~0 != candidate_range_bi)
 937     {
 938       vlib_buffer_t *candidate_b = vlib_get_buffer (vm, candidate_range_bi);
 939       vnet_buffer_opaque_t *candidate_vnb = vnet_buffer (candidate_b);
 940       if (fragment_first > candidate_vnb->ip.reass.range_last)
 941         {
 942           // this fragments starts after candidate range
 943           prev_range_bi = candidate_range_bi;
 944           candidate_range_bi = candidate_vnb->ip.reass.next_range_bi;
 945           if (candidate_vnb->ip.reass.range_last < fragment_last &&
 946               ~0 == candidate_range_bi)
 947             {
 948               // special case - this fragment falls beyond all known ranges
 949               ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
 950                                                     *bi0);
 951               consumed = 1;
 952               break;
 953             }
 954           continue;
 955         }
 956       if (fragment_last < candidate_vnb->ip.reass.range_first)
 957         {
 958           // this fragment ends before candidate range without any overlap
 959           ip6_full_reass_insert_range_in_chain (vm, reass, prev_range_bi,
 960                                                 *bi0);
 961           consumed = 1;
 962         }
 963       else if (fragment_first == candidate_vnb->ip.reass.range_first &&
 964                fragment_last == candidate_vnb->ip.reass.range_last)
 965         {
 966           // duplicate fragment - ignore
 967         }
 968       else
 969         {
 970           // overlapping fragment - not allowed by RFC 8200
 971           if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 972             {
 973               ip6_full_reass_add_trace (vm, node, reass, *bi0, frag_hdr,
 974                                         RANGE_OVERLAP, ~0);
 975             }
 976           return IP6_FULL_REASS_RC_OVERLAP;
 977         }
 978       break;
 979     }
 980   ++reass->fragments_n;
 981 check_if_done_maybe:
 982   if (consumed)
 983     {
 984       if (PREDICT_FALSE (fb->flags & VLIB_BUFFER_IS_TRACED))
 985         {
 986           ip6_full_reass_add_trace (vm, node, reass, *bi0, frag_hdr, RANGE_NEW,
 987                                     ~0);
 988         }
 989     }
 990   else if (skip_bihash)
 991     {
 992       // if this reassembly is not in bihash, then the packet must have been
 993       // consumed
 994       return IP6_FULL_REASS_RC_INTERNAL_ERROR;
 995     }
 996   if (~0 != reass->last_packet_octet &&
 997       reass->data_len == reass->last_packet_octet + 1)
 998     {
 999       *handoff_thread_idx = reass->sendout_thread_index;
1000       int handoff =
1001         reass->memory_owner_thread_index != reass->sendout_thread_index;
1002       ip6_full_reass_rc_t rc =
1003         ip6_full_reass_finalize (vm, node, rm, rt, reass, bi0, next0, error0,
1004                                  is_custom_app);
1005       if (IP6_FULL_REASS_RC_OK == rc && handoff)
1006         {
1007           return IP6_FULL_REASS_RC_HANDOFF;
1008         }
1009       return rc;
1010     }
1011   else
1012     {
1013       if (skip_bihash)
1014         {
1015           // if this reassembly is not in bihash, it should've been an atomic
1016           // fragment and thus finalized
1017           return IP6_FULL_REASS_RC_INTERNAL_ERROR;
1018         }
1019       if (consumed)
1020         {
1021           *bi0 = ~0;
1022           if (reass->fragments_n > rm->max_reass_len)
1023             {
1024               return IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS;
1025             }
1026         }
1027       else
1028         {
1029           *next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1030           *error0 = IP6_ERROR_REASS_DUPLICATE_FRAGMENT;
1031         }
1032     }
1033   return IP6_FULL_REASS_RC_OK;
1034 }
1035
1036 always_inline bool
1037 ip6_full_reass_verify_upper_layer_present (vlib_node_runtime_t *node,
1038                                            vlib_buffer_t *b,
1039                                            ip6_ext_hdr_chain_t *hc)
1040 {
1041   int nh = hc->eh[hc->length - 1].protocol;
1042   /* Checking to see if it's a terminating header */
1043   if (ip6_ext_hdr (nh))
1044     {
1045       icmp6_error_set_vnet_buffer (
1046         b, ICMP6_parameter_problem,
1047         ICMP6_parameter_problem_first_fragment_has_incomplete_header_chain, 0);
1048       b->error = node->errors[IP6_ERROR_REASS_MISSING_UPPER];
1049       return false;
1050     }
1051   return true;
1052 }
1053
1054 always_inline bool
1055 ip6_full_reass_verify_fragment_multiple_8 (vlib_main_t * vm,
1056                                            vlib_buffer_t * b,
1057                                            ip6_frag_hdr_t * frag_hdr)
1058 {
1059   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1060   ip6_header_t *ip = vlib_buffer_get_current (b);
1061   int more_fragments = ip6_frag_hdr_more (frag_hdr);
1062   u32 fragment_length =
1063     vlib_buffer_length_in_chain (vm, b) -
1064     (vnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
1065   if (more_fragments && 0 != fragment_length % 8)
1066     {
1067       icmp6_error_set_vnet_buffer (b, ICMP6_parameter_problem,
1068                                    ICMP6_parameter_problem_erroneous_header_field,
1069                                    (u8 *) & ip->payload_length - (u8 *) ip);
1070       return false;
1071     }
1072   return true;
1073 }
1074
1075 always_inline bool
1076 ip6_full_reass_verify_packet_size_lt_64k (vlib_main_t * vm,
1077                                           vlib_buffer_t * b,
1078                                           ip6_frag_hdr_t * frag_hdr)
1079 {
1080   vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1081   u32 fragment_first = ip6_frag_hdr_offset_bytes (frag_hdr);
1082   u32 fragment_length =
1083     vlib_buffer_length_in_chain (vm, b) -
1084     (vnb->ip.reass.ip6_frag_hdr_offset + sizeof (*frag_hdr));
1085   if (fragment_first + fragment_length > 65535)
1086     {
1087       ip6_header_t *ip0 = vlib_buffer_get_current (b);
1088       icmp6_error_set_vnet_buffer (b, ICMP6_parameter_problem,
1089                                    ICMP6_parameter_problem_erroneous_header_field,
1090                                    (u8 *) & frag_hdr->fragment_offset_and_more
1091                                    - (u8 *) ip0);
1092       return false;
1093     }
1094   return true;
1095 }
1096
1097 always_inline uword
1098 ip6_full_reassembly_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1099                             vlib_frame_t *frame, bool is_feature,
1100                             bool is_custom_app, bool is_local)
1101 {
1102   u32 *from = vlib_frame_vector_args (frame);
1103   u32 n_left_from, n_left_to_next, *to_next, next_index;
1104   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1105   ip6_full_reass_per_thread_t *rt = &rm->per_thread_data[vm->thread_index];
1106   clib_spinlock_lock (&rt->lock);
1107
1108   n_left_from = frame->n_vectors;
1109   next_index = node->cached_next_index;
1110   while (n_left_from > 0)
1111     {
1112       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1113
1114       while (n_left_from > 0 && n_left_to_next > 0)
1115         {
1116           u32 bi0;
1117           vlib_buffer_t *b0;
1118           u32 next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1119           u32 error0 = IP6_ERROR_NONE;
1120           u32 icmp_bi = ~0;
1121
1122           bi0 = from[0];
1123           b0 = vlib_get_buffer (vm, bi0);
1124
1125           ip6_header_t *ip0 = vlib_buffer_get_current (b0);
1126           ip6_frag_hdr_t *frag_hdr;
1127           ip6_ext_hdr_chain_t hdr_chain;
1128           int res = ip6_ext_header_walk (
1129             b0, ip0, IP_PROTOCOL_IPV6_FRAGMENTATION, &hdr_chain);
1130           if (res < 0 ||
1131               hdr_chain.eh[res].protocol != IP_PROTOCOL_IPV6_FRAGMENTATION)
1132             {
1133               // this is a mangled packet - no fragmentation
1134               next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1135               ip6_full_reass_add_trace (vm, node, NULL, bi0, NULL, PASSTHROUGH,
1136                                         ~0);
1137               goto skip_reass;
1138             }
1139           if (is_local && !rm->is_local_reass_enabled)
1140             {
1141               next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1142               goto skip_reass;
1143             }
1144           frag_hdr =
1145             ip6_ext_next_header_offset (ip0, hdr_chain.eh[res].offset);
1146           vnet_buffer (b0)->ip.reass.ip6_frag_hdr_offset =
1147             hdr_chain.eh[res].offset;
1148
1149           if (0 == ip6_frag_hdr_offset (frag_hdr))
1150             {
1151               // first fragment - verify upper-layer is present
1152               if (!ip6_full_reass_verify_upper_layer_present (node, b0,
1153                                                               &hdr_chain))
1154                 {
1155                   next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1156                   goto skip_reass;
1157                 }
1158             }
1159           if (!ip6_full_reass_verify_fragment_multiple_8 (vm, b0, frag_hdr) ||
1160               !ip6_full_reass_verify_packet_size_lt_64k (vm, b0, frag_hdr))
1161             {
1162               next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1163               goto skip_reass;
1164             }
1165
1166           int skip_bihash = 0;
1167           ip6_full_reass_kv_t kv;
1168           u8 do_handoff = 0;
1169
1170           if (0 == ip6_frag_hdr_offset (frag_hdr) &&
1171               !ip6_frag_hdr_more (frag_hdr))
1172             {
1173               // this is atomic fragment and needs to be processed separately
1174               skip_bihash = 1;
1175             }
1176           else
1177             {
1178               kv.k.as_u64[0] = ip0->src_address.as_u64[0];
1179               kv.k.as_u64[1] = ip0->src_address.as_u64[1];
1180               kv.k.as_u64[2] = ip0->dst_address.as_u64[0];
1181               kv.k.as_u64[3] = ip0->dst_address.as_u64[1];
1182               kv.k.as_u64[4] =
1183                 ((u64) vec_elt (ip6_main.fib_index_by_sw_if_index,
1184                                 vnet_buffer (b0)->sw_if_index[VLIB_RX]))
1185                   << 32 |
1186                 (u64) frag_hdr->identification;
1187               kv.k.as_u64[5] = ip0->protocol;
1188             }
1189
1190           ip6_full_reass_t *reass = ip6_full_reass_find_or_create (
1191             vm, node, rm, rt, &kv, &icmp_bi, &do_handoff, skip_bihash);
1192
1193           if (reass)
1194             {
1195               const u32 fragment_first = ip6_frag_hdr_offset (frag_hdr);
1196               if (0 == fragment_first)
1197                 {
1198                   reass->sendout_thread_index = vm->thread_index;
1199                 }
1200             }
1201           if (PREDICT_FALSE (do_handoff))
1202             {
1203               next0 = IP6_FULL_REASSEMBLY_NEXT_HANDOFF;
1204               vnet_buffer (b0)->ip.reass.owner_thread_index =
1205                 kv.v.memory_owner_thread_index;
1206             }
1207           else if (reass)
1208             {
1209               u32 handoff_thread_idx;
1210               u32 counter = ~0;
1211               switch (ip6_full_reass_update (
1212                 vm, node, rm, rt, reass, &bi0, &next0, &error0, frag_hdr,
1213                 is_custom_app, &handoff_thread_idx, skip_bihash))
1214                 {
1215                 case IP6_FULL_REASS_RC_OK:
1216                   /* nothing to do here */
1217                   break;
1218                 case IP6_FULL_REASS_RC_HANDOFF:
1219                   next0 = IP6_FULL_REASSEMBLY_NEXT_HANDOFF;
1220                   b0 = vlib_get_buffer (vm, bi0);
1221                   vnet_buffer (b0)->ip.reass.owner_thread_index =
1222                     handoff_thread_idx;
1223                   break;
1224                 case IP6_FULL_REASS_RC_TOO_MANY_FRAGMENTS:
1225                   counter = IP6_ERROR_REASS_FRAGMENT_CHAIN_TOO_LONG;
1226                   break;
1227                 case IP6_FULL_REASS_RC_NO_BUF:
1228                   counter = IP6_ERROR_REASS_NO_BUF;
1229                   break;
1230                 case IP6_FULL_REASS_RC_INTERNAL_ERROR:
1231                   counter = IP6_ERROR_REASS_INTERNAL_ERROR;
1232                   break;
1233                 case IP6_FULL_REASS_RC_INVALID_FRAG_LEN:
1234                   counter = IP6_ERROR_REASS_INVALID_FRAG_LEN;
1235                   break;
1236                 case IP6_FULL_REASS_RC_OVERLAP:
1237                   counter = IP6_ERROR_REASS_OVERLAPPING_FRAGMENT;
1238                   break;
1239                 }
1240               if (~0 != counter)
1241                 {
1242                   vlib_node_increment_counter (vm, node->node_index, counter,
1243                                                1);
1244                   ip6_full_reass_drop_all (vm, node, reass, bi0);
1245                   ip6_full_reass_free (rm, rt, reass);
1246                   goto next_packet;
1247                   break;
1248                 }
1249             }
1250           else
1251             {
1252               if (is_feature)
1253                 {
1254                   next0 = IP6_FULL_REASSEMBLY_NEXT_DROP;
1255                 }
1256               else
1257                 {
1258                   vnet_buffer_opaque_t *fvnb = vnet_buffer (b0);
1259                   next0 = fvnb->ip.reass.error_next_index;
1260                 }
1261               error0 = IP6_ERROR_REASS_LIMIT_REACHED;
1262             }
1263
1264           if (~0 != bi0)
1265             {
1266             skip_reass:
1267               to_next[0] = bi0;
1268               to_next += 1;
1269               n_left_to_next -= 1;
1270
1271               /* bi0 might have been updated by reass_finalize, reload */
1272               b0 = vlib_get_buffer (vm, bi0);
1273               if (IP6_ERROR_NONE != error0)
1274                 {
1275                   b0->error = node->errors[error0];
1276                 }
1277
1278               if (next0 == IP6_FULL_REASSEMBLY_NEXT_HANDOFF)
1279                 {
1280                   if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
1281                     {
1282                       ip6_full_reass_add_trace (
1283                         vm, node, NULL, bi0, frag_hdr, HANDOFF,
1284                         vnet_buffer (b0)->ip.reass.owner_thread_index);
1285                     }
1286                 }
1287               else if (is_feature && IP6_ERROR_NONE == error0)
1288                 {
1289                   vnet_feature_next (&next0, b0);
1290                 }
1291               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1292                                                n_left_to_next, bi0, next0);
1293             }
1294
1295           if (~0 != icmp_bi)
1296             {
1297               next0 = IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR;
1298               to_next[0] = icmp_bi;
1299               to_next += 1;
1300               n_left_to_next -= 1;
1301               vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
1302                                                n_left_to_next, icmp_bi,
1303                                                next0);
1304             }
1305         next_packet:
1306           from += 1;
1307           n_left_from -= 1;
1308         }
1309
1310       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1311     }
1312
1313   clib_spinlock_unlock (&rt->lock);
1314   return frame->n_vectors;
1315 }
1316
1317 static char *ip6_full_reassembly_error_strings[] = {
1318 #define _(sym, string) string,
1319   foreach_ip6_error
1320 #undef _
1321 };
1322
1323 VLIB_NODE_FN (ip6_full_reass_node) (vlib_main_t * vm,
1324                                     vlib_node_runtime_t * node,
1325                                     vlib_frame_t * frame)
1326 {
1327   return ip6_full_reassembly_inline (vm, node, frame, false /* is_feature */,
1328                                      false /* is_custom_app */,
1329                                      false /* is_local */);
1330 }
1331
1332 VLIB_REGISTER_NODE (ip6_full_reass_node) = {
1333     .name = "ip6-full-reassembly",
1334     .vector_size = sizeof (u32),
1335     .format_trace = format_ip6_full_reass_trace,
1336     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1337     .error_strings = ip6_full_reassembly_error_strings,
1338     .n_next_nodes = IP6_FULL_REASSEMBLY_N_NEXT,
1339     .next_nodes =
1340         {
1341                 [IP6_FULL_REASSEMBLY_NEXT_INPUT] = "ip6-input",
1342                 [IP6_FULL_REASSEMBLY_NEXT_DROP] = "ip6-drop",
1343                 [IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR] = "ip6-icmp-error",
1344                 [IP6_FULL_REASSEMBLY_NEXT_HANDOFF] = "ip6-full-reassembly-handoff",
1345         },
1346 };
1347
1348 VLIB_NODE_FN (ip6_local_full_reass_node)
1349 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1350 {
1351   return ip6_full_reassembly_inline (vm, node, frame, false /* is_feature */,
1352                                      false /* is_custom_app */,
1353                                      true /* is_local */);
1354 }
1355
1356 VLIB_REGISTER_NODE (ip6_local_full_reass_node) = {
1357     .name = "ip6-local-full-reassembly",
1358     .vector_size = sizeof (u32),
1359     .format_trace = format_ip6_full_reass_trace,
1360     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1361     .error_strings = ip6_full_reassembly_error_strings,
1362     .n_next_nodes = IP6_FULL_REASSEMBLY_N_NEXT,
1363     .next_nodes =
1364         {
1365                 [IP6_FULL_REASSEMBLY_NEXT_INPUT] = "ip6-input",
1366                 [IP6_FULL_REASSEMBLY_NEXT_DROP] = "ip6-drop",
1367                 [IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR] = "ip6-icmp-error",
1368                 [IP6_FULL_REASSEMBLY_NEXT_HANDOFF] = "ip6-local-full-reassembly-handoff",
1369         },
1370 };
1371
1372 VLIB_NODE_FN (ip6_full_reass_node_feature) (vlib_main_t * vm,
1373                                             vlib_node_runtime_t * node,
1374                                             vlib_frame_t * frame)
1375 {
1376   return ip6_full_reassembly_inline (vm, node, frame, true /* is_feature */,
1377                                      false /* is_custom_app */,
1378                                      false /* is_local */);
1379 }
1380
1381 VLIB_REGISTER_NODE (ip6_full_reass_node_feature) = {
1382     .name = "ip6-full-reassembly-feature",
1383     .vector_size = sizeof (u32),
1384     .format_trace = format_ip6_full_reass_trace,
1385     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1386     .error_strings = ip6_full_reassembly_error_strings,
1387     .n_next_nodes = IP6_FULL_REASSEMBLY_N_NEXT,
1388     .next_nodes =
1389         {
1390                 [IP6_FULL_REASSEMBLY_NEXT_INPUT] = "ip6-input",
1391                 [IP6_FULL_REASSEMBLY_NEXT_DROP] = "ip6-drop",
1392                 [IP6_FULL_REASSEMBLY_NEXT_ICMP_ERROR] = "ip6-icmp-error",
1393                 [IP6_FULL_REASSEMBLY_NEXT_HANDOFF] = "ip6-full-reass-feature-hoff",
1394         },
1395 };
1396
1397 VNET_FEATURE_INIT (ip6_full_reassembly_feature, static) = {
1398     .arc_name = "ip6-unicast",
1399     .node_name = "ip6-full-reassembly-feature",
1400     .runs_before = VNET_FEATURES ("ip6-lookup",
1401                                   "ipsec6-input-feature"),
1402     .runs_after = 0,
1403 };
1404
1405 #ifndef CLIB_MARCH_VARIANT
1406 static u32
1407 ip6_full_reass_get_nbuckets ()
1408 {
1409   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1410   u32 nbuckets;
1411   u8 i;
1412
1413   nbuckets = (u32) (rm->max_reass_n / IP6_FULL_REASS_HT_LOAD_FACTOR);
1414
1415   for (i = 0; i < 31; i++)
1416     if ((1 << i) >= nbuckets)
1417       break;
1418   nbuckets = 1 << i;
1419
1420   return nbuckets;
1421 }
1422 #endif /* CLIB_MARCH_VARIANT */
1423
1424 typedef enum
1425 {
1426   IP6_EVENT_CONFIG_CHANGED = 1,
1427 } ip6_full_reass_event_t;
1428
1429 #ifndef CLIB_MARCH_VARIANT
1430 typedef struct
1431 {
1432   int failure;
1433   clib_bihash_48_8_t *new_hash;
1434 } ip6_rehash_cb_ctx;
1435
1436 static int
1437 ip6_rehash_cb (clib_bihash_kv_48_8_t * kv, void *_ctx)
1438 {
1439   ip6_rehash_cb_ctx *ctx = _ctx;
1440   if (clib_bihash_add_del_48_8 (ctx->new_hash, kv, 1))
1441     {
1442       ctx->failure = 1;
1443     }
1444   return (BIHASH_WALK_CONTINUE);
1445 }
1446
1447 static void
1448 ip6_full_reass_set_params (u32 timeout_ms, u32 max_reassemblies,
1449                            u32 max_reassembly_length,
1450                            u32 expire_walk_interval_ms)
1451 {
1452   ip6_full_reass_main.timeout_ms = timeout_ms;
1453   ip6_full_reass_main.timeout = (f64) timeout_ms / (f64) MSEC_PER_SEC;
1454   ip6_full_reass_main.max_reass_n = max_reassemblies;
1455   ip6_full_reass_main.max_reass_len = max_reassembly_length;
1456   ip6_full_reass_main.expire_walk_interval_ms = expire_walk_interval_ms;
1457 }
1458
1459 vnet_api_error_t
1460 ip6_full_reass_set (u32 timeout_ms, u32 max_reassemblies,
1461                     u32 max_reassembly_length, u32 expire_walk_interval_ms)
1462 {
1463   u32 old_nbuckets = ip6_full_reass_get_nbuckets ();
1464   ip6_full_reass_set_params (timeout_ms, max_reassemblies,
1465                              max_reassembly_length, expire_walk_interval_ms);
1466   vlib_process_signal_event (ip6_full_reass_main.vlib_main,
1467                              ip6_full_reass_main.ip6_full_reass_expire_node_idx,
1468                              IP6_EVENT_CONFIG_CHANGED, 0);
1469   u32 new_nbuckets = ip6_full_reass_get_nbuckets ();
1470   if (ip6_full_reass_main.max_reass_n > 0 && new_nbuckets > old_nbuckets)
1471     {
1472       clib_bihash_48_8_t new_hash;
1473       clib_memset (&new_hash, 0, sizeof (new_hash));
1474       ip6_rehash_cb_ctx ctx;
1475       ctx.failure = 0;
1476       ctx.new_hash = &new_hash;
1477       clib_bihash_init_48_8 (&new_hash, "ip6-full-reass", new_nbuckets,
1478                              new_nbuckets * 1024);
1479       clib_bihash_foreach_key_value_pair_48_8 (&ip6_full_reass_main.hash,
1480                                                ip6_rehash_cb, &ctx);
1481       if (ctx.failure)
1482         {
1483           clib_bihash_free_48_8 (&new_hash);
1484           return -1;
1485         }
1486       else
1487         {
1488           clib_bihash_free_48_8 (&ip6_full_reass_main.hash);
1489           clib_memcpy_fast (&ip6_full_reass_main.hash, &new_hash,
1490                             sizeof (ip6_full_reass_main.hash));
1491           clib_bihash_copied (&ip6_full_reass_main.hash, &new_hash);
1492         }
1493     }
1494   return 0;
1495 }
1496
1497 vnet_api_error_t
1498 ip6_full_reass_get (u32 * timeout_ms, u32 * max_reassemblies,
1499                     u32 * max_reassembly_length,
1500                     u32 * expire_walk_interval_ms)
1501 {
1502   *timeout_ms = ip6_full_reass_main.timeout_ms;
1503   *max_reassemblies = ip6_full_reass_main.max_reass_n;
1504   *max_reassembly_length = ip6_full_reass_main.max_reass_len;
1505   *expire_walk_interval_ms = ip6_full_reass_main.expire_walk_interval_ms;
1506   return 0;
1507 }
1508
1509 static clib_error_t *
1510 ip6_full_reass_init_function (vlib_main_t * vm)
1511 {
1512   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1513   clib_error_t *error = 0;
1514   u32 nbuckets;
1515   vlib_node_t *node;
1516
1517   rm->vlib_main = vm;
1518
1519   vec_validate (rm->per_thread_data, vlib_num_workers ());
1520   ip6_full_reass_per_thread_t *rt;
1521   vec_foreach (rt, rm->per_thread_data)
1522   {
1523     clib_spinlock_init (&rt->lock);
1524     pool_alloc (rt->pool, rm->max_reass_n);
1525   }
1526
1527   node = vlib_get_node_by_name (vm, (u8 *) "ip6-full-reassembly-expire-walk");
1528   ASSERT (node);
1529   rm->ip6_full_reass_expire_node_idx = node->index;
1530
1531   ip6_full_reass_set_params (IP6_FULL_REASS_TIMEOUT_DEFAULT_MS,
1532                              IP6_FULL_REASS_MAX_REASSEMBLIES_DEFAULT,
1533                              IP6_FULL_REASS_MAX_REASSEMBLY_LENGTH_DEFAULT,
1534                              IP6_FULL_REASS_EXPIRE_WALK_INTERVAL_DEFAULT_MS);
1535
1536   nbuckets = ip6_full_reass_get_nbuckets ();
1537   clib_bihash_init_48_8 (&rm->hash, "ip6-full-reass", nbuckets,
1538                          nbuckets * 1024);
1539
1540   node = vlib_get_node_by_name (vm, (u8 *) "ip6-icmp-error");
1541   ASSERT (node);
1542   rm->ip6_icmp_error_idx = node->index;
1543
1544   if ((error = vlib_call_init_function (vm, ip_main_init)))
1545     return error;
1546   ip6_register_protocol (IP_PROTOCOL_IPV6_FRAGMENTATION,
1547                          ip6_local_full_reass_node.index);
1548   rm->is_local_reass_enabled = 1;
1549
1550   rm->fq_index = vlib_frame_queue_main_init (ip6_full_reass_node.index, 0);
1551   rm->fq_local_index =
1552     vlib_frame_queue_main_init (ip6_local_full_reass_node.index, 0);
1553   rm->fq_feature_index =
1554     vlib_frame_queue_main_init (ip6_full_reass_node_feature.index, 0);
1555
1556   rm->feature_use_refcount_per_intf = NULL;
1557   return error;
1558 }
1559
1560 VLIB_INIT_FUNCTION (ip6_full_reass_init_function);
1561 #endif /* CLIB_MARCH_VARIANT */
1562
1563 static uword
1564 ip6_full_reass_walk_expired (vlib_main_t *vm, vlib_node_runtime_t *node,
1565                              CLIB_UNUSED (vlib_frame_t *f))
1566 {
1567   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1568   uword event_type, *event_data = 0;
1569
1570   while (true)
1571     {
1572       vlib_process_wait_for_event_or_clock (vm,
1573                                             (f64) rm->expire_walk_interval_ms
1574                                             / (f64) MSEC_PER_SEC);
1575       event_type = vlib_process_get_events (vm, &event_data);
1576
1577       switch (event_type)
1578         {
1579         case ~0:
1580           /* no events => timeout */
1581           /* fallthrough */
1582         case IP6_EVENT_CONFIG_CHANGED:
1583           /* nothing to do here */
1584           break;
1585         default:
1586           clib_warning ("BUG: event type 0x%wx", event_type);
1587           break;
1588         }
1589       f64 now = vlib_time_now (vm);
1590
1591       ip6_full_reass_t *reass;
1592       int *pool_indexes_to_free = NULL;
1593
1594       uword thread_index = 0;
1595       int index;
1596       const uword nthreads = vlib_num_workers () + 1;
1597       u32 *vec_icmp_bi = NULL;
1598       for (thread_index = 0; thread_index < nthreads; ++thread_index)
1599         {
1600           ip6_full_reass_per_thread_t *rt =
1601             &rm->per_thread_data[thread_index];
1602           clib_spinlock_lock (&rt->lock);
1603
1604           vec_reset_length (pool_indexes_to_free);
1605           pool_foreach_index (index, rt->pool)  {
1606                                 reass = pool_elt_at_index (rt->pool, index);
1607                                 if (now > reass->last_heard + rm->timeout)
1608                                   {
1609                                     vec_add1 (pool_indexes_to_free, index);
1610                                   }
1611                               }
1612           int *i;
1613           vec_foreach (i, pool_indexes_to_free)
1614           {
1615             ip6_full_reass_t *reass = pool_elt_at_index (rt->pool, i[0]);
1616             u32 icmp_bi = ~0;
1617             ip6_full_reass_on_timeout (vm, node, reass, &icmp_bi);
1618             if (~0 != icmp_bi)
1619               vec_add1 (vec_icmp_bi, icmp_bi);
1620
1621             ip6_full_reass_free (rm, rt, reass);
1622           }
1623
1624           clib_spinlock_unlock (&rt->lock);
1625         }
1626
1627       while (vec_len (vec_icmp_bi) > 0)
1628         {
1629           vlib_frame_t *f =
1630             vlib_get_frame_to_node (vm, rm->ip6_icmp_error_idx);
1631           u32 *to_next = vlib_frame_vector_args (f);
1632           u32 n_left_to_next = VLIB_FRAME_SIZE - f->n_vectors;
1633           int trace_frame = 0;
1634           while (vec_len (vec_icmp_bi) > 0 && n_left_to_next > 0)
1635             {
1636               u32 bi = vec_pop (vec_icmp_bi);
1637               vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1638               if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1639                 trace_frame = 1;
1640               b->error = node->errors[IP6_ERROR_REASS_TIMEOUT];
1641               to_next[0] = bi;
1642               ++f->n_vectors;
1643               to_next += 1;
1644               n_left_to_next -= 1;
1645             }
1646           f->frame_flags |= (trace_frame * VLIB_FRAME_TRACE);
1647           vlib_put_frame_to_node (vm, rm->ip6_icmp_error_idx, f);
1648         }
1649
1650       vec_free (pool_indexes_to_free);
1651       vec_free (vec_icmp_bi);
1652       if (event_data)
1653         {
1654           vec_set_len (event_data, 0);
1655         }
1656     }
1657
1658   return 0;
1659 }
1660
1661 VLIB_REGISTER_NODE (ip6_full_reass_expire_node) = {
1662     .function = ip6_full_reass_walk_expired,
1663     .format_trace = format_ip6_full_reass_trace,
1664     .type = VLIB_NODE_TYPE_PROCESS,
1665     .name = "ip6-full-reassembly-expire-walk",
1666
1667     .n_errors = ARRAY_LEN (ip6_full_reassembly_error_strings),
1668     .error_strings = ip6_full_reassembly_error_strings,
1669
1670 };
1671
1672 static u8 *
1673 format_ip6_full_reass_key (u8 * s, va_list * args)
1674 {
1675   ip6_full_reass_key_t *key = va_arg (*args, ip6_full_reass_key_t *);
1676   s = format (s, "xx_id: %u, src: %U, dst: %U, frag_id: %u, proto: %u",
1677               key->xx_id, format_ip6_address, &key->src, format_ip6_address,
1678               &key->dst, clib_net_to_host_u16 (key->frag_id), key->proto);
1679   return s;
1680 }
1681
1682 static u8 *
1683 format_ip6_full_reass (u8 * s, va_list * args)
1684 {
1685   vlib_main_t *vm = va_arg (*args, vlib_main_t *);
1686   ip6_full_reass_t *reass = va_arg (*args, ip6_full_reass_t *);
1687
1688   s = format (s, "ID: %lu, key: %U\n  first_bi: %u, data_len: %u, "
1689               "last_packet_octet: %u, trace_op_counter: %u\n",
1690               reass->id, format_ip6_full_reass_key, &reass->key,
1691               reass->first_bi, reass->data_len, reass->last_packet_octet,
1692               reass->trace_op_counter);
1693   u32 bi = reass->first_bi;
1694   u32 counter = 0;
1695   while (~0 != bi)
1696     {
1697       vlib_buffer_t *b = vlib_get_buffer (vm, bi);
1698       vnet_buffer_opaque_t *vnb = vnet_buffer (b);
1699       s = format (s, "  #%03u: range: [%u, %u], bi: %u, off: %d, len: %u, "
1700                   "fragment[%u, %u]\n",
1701                   counter, vnb->ip.reass.range_first,
1702                   vnb->ip.reass.range_last, bi,
1703                   ip6_full_reass_buffer_get_data_offset (b),
1704                   ip6_full_reass_buffer_get_data_len (b),
1705                   vnb->ip.reass.fragment_first, vnb->ip.reass.fragment_last);
1706       if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1707         {
1708           bi = b->next_buffer;
1709         }
1710       else
1711         {
1712           bi = ~0;
1713         }
1714     }
1715   return s;
1716 }
1717
1718 static clib_error_t *
1719 show_ip6_full_reass (vlib_main_t * vm, unformat_input_t * input,
1720                      CLIB_UNUSED (vlib_cli_command_t * lmd))
1721 {
1722   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1723
1724   vlib_cli_output (vm, "---------------------");
1725   vlib_cli_output (vm, "IP6 reassembly status");
1726   vlib_cli_output (vm, "---------------------");
1727   bool details = false;
1728   if (unformat (input, "details"))
1729     {
1730       details = true;
1731     }
1732
1733   u32 sum_reass_n = 0;
1734   u64 sum_buffers_n = 0;
1735   ip6_full_reass_t *reass;
1736   uword thread_index;
1737   const uword nthreads = vlib_num_workers () + 1;
1738   for (thread_index = 0; thread_index < nthreads; ++thread_index)
1739     {
1740       ip6_full_reass_per_thread_t *rt = &rm->per_thread_data[thread_index];
1741       clib_spinlock_lock (&rt->lock);
1742       if (details)
1743         {
1744           pool_foreach (reass, rt->pool) {
1745             vlib_cli_output (vm, "%U", format_ip6_full_reass, vm, reass);
1746           }
1747         }
1748       sum_reass_n += rt->reass_n;
1749       clib_spinlock_unlock (&rt->lock);
1750     }
1751   vlib_cli_output (vm, "---------------------");
1752   vlib_cli_output (vm, "Current IP6 reassemblies count: %lu\n",
1753                    (long unsigned) sum_reass_n);
1754   vlib_cli_output (vm,
1755                    "Maximum configured concurrent full IP6 reassemblies per worker-thread: %lu\n",
1756                    (long unsigned) rm->max_reass_n);
1757   vlib_cli_output (vm,
1758                    "Maximum configured amount of fragments "
1759                    "per full IP6 reassembly: %lu\n",
1760                    (long unsigned) rm->max_reass_len);
1761   vlib_cli_output (vm,
1762                    "Maximum configured full IP6 reassembly timeout: %lums\n",
1763                    (long unsigned) rm->timeout_ms);
1764   vlib_cli_output (vm,
1765                    "Maximum configured full IP6 reassembly expire walk interval: %lums\n",
1766                    (long unsigned) rm->expire_walk_interval_ms);
1767   vlib_cli_output (vm, "Buffers in use: %lu\n",
1768                    (long unsigned) sum_buffers_n);
1769   return 0;
1770 }
1771
1772 VLIB_CLI_COMMAND (show_ip6_full_reassembly_cmd, static) = {
1773     .path = "show ip6-full-reassembly",
1774     .short_help = "show ip6-full-reassembly [details]",
1775     .function = show_ip6_full_reass,
1776 };
1777
1778 #ifndef CLIB_MARCH_VARIANT
1779 vnet_api_error_t
1780 ip6_full_reass_enable_disable (u32 sw_if_index, u8 enable_disable)
1781 {
1782   return vnet_feature_enable_disable ("ip6-unicast",
1783                                       "ip6-full-reassembly-feature",
1784                                       sw_if_index, enable_disable, 0, 0);
1785 }
1786 #endif /* CLIB_MARCH_VARIANT */
1787
1788 #define foreach_ip6_full_reassembly_handoff_error                       \
1789 _(CONGESTION_DROP, "congestion drop")
1790
1791
1792 typedef enum
1793 {
1794 #define _(sym,str) IP6_FULL_REASSEMBLY_HANDOFF_ERROR_##sym,
1795   foreach_ip6_full_reassembly_handoff_error
1796 #undef _
1797     IP6_FULL_REASSEMBLY_HANDOFF_N_ERROR,
1798 } ip6_full_reassembly_handoff_error_t;
1799
1800 static char *ip6_full_reassembly_handoff_error_strings[] = {
1801 #define _(sym,string) string,
1802   foreach_ip6_full_reassembly_handoff_error
1803 #undef _
1804 };
1805
1806 typedef struct
1807 {
1808   u32 next_worker_index;
1809 } ip6_full_reassembly_handoff_trace_t;
1810
1811 static u8 *
1812 format_ip6_full_reassembly_handoff_trace (u8 * s, va_list * args)
1813 {
1814   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1815   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1816   ip6_full_reassembly_handoff_trace_t *t =
1817     va_arg (*args, ip6_full_reassembly_handoff_trace_t *);
1818
1819   s =
1820     format (s, "ip6-full-reassembly-handoff: next-worker %d",
1821             t->next_worker_index);
1822
1823   return s;
1824 }
1825
1826 always_inline uword
1827 ip6_full_reassembly_handoff_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1828                                     vlib_frame_t *frame, bool is_feature,
1829                                     bool is_local)
1830 {
1831   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1832
1833   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
1834   u32 n_enq, n_left_from, *from;
1835   u16 thread_indices[VLIB_FRAME_SIZE], *ti;
1836   u32 fq_index;
1837
1838   from = vlib_frame_vector_args (frame);
1839   n_left_from = frame->n_vectors;
1840   vlib_get_buffers (vm, from, bufs, n_left_from);
1841
1842   b = bufs;
1843   ti = thread_indices;
1844
1845   if (is_feature)
1846     {
1847       fq_index = rm->fq_feature_index;
1848     }
1849   else
1850     {
1851       if (is_local)
1852         {
1853           fq_index = rm->fq_local_index;
1854         }
1855       else
1856         {
1857           fq_index = rm->fq_index;
1858         }
1859     }
1860
1861   while (n_left_from > 0)
1862     {
1863       ti[0] = vnet_buffer (b[0])->ip.reass.owner_thread_index;
1864
1865       if (PREDICT_FALSE
1866           ((node->flags & VLIB_NODE_FLAG_TRACE)
1867            && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
1868         {
1869           ip6_full_reassembly_handoff_trace_t *t =
1870             vlib_add_trace (vm, node, b[0], sizeof (*t));
1871           t->next_worker_index = ti[0];
1872         }
1873
1874       n_left_from -= 1;
1875       ti += 1;
1876       b += 1;
1877     }
1878   n_enq = vlib_buffer_enqueue_to_thread (vm, node, fq_index, from,
1879                                          thread_indices, frame->n_vectors, 1);
1880
1881   if (n_enq < frame->n_vectors)
1882     vlib_node_increment_counter (vm, node->node_index,
1883                                  IP6_FULL_REASSEMBLY_HANDOFF_ERROR_CONGESTION_DROP,
1884                                  frame->n_vectors - n_enq);
1885   return frame->n_vectors;
1886 }
1887
1888 VLIB_NODE_FN (ip6_full_reassembly_handoff_node) (vlib_main_t * vm,
1889                                                  vlib_node_runtime_t * node,
1890                                                  vlib_frame_t * frame)
1891 {
1892   return ip6_full_reassembly_handoff_inline (
1893     vm, node, frame, false /* is_feature */, false /* is_local */);
1894 }
1895
1896 VLIB_REGISTER_NODE (ip6_full_reassembly_handoff_node) = {
1897   .name = "ip6-full-reassembly-handoff",
1898   .vector_size = sizeof (u32),
1899   .n_errors = ARRAY_LEN(ip6_full_reassembly_handoff_error_strings),
1900   .error_strings = ip6_full_reassembly_handoff_error_strings,
1901   .format_trace = format_ip6_full_reassembly_handoff_trace,
1902
1903   .n_next_nodes = 1,
1904
1905   .next_nodes = {
1906     [0] = "error-drop",
1907   },
1908 };
1909
1910 VLIB_NODE_FN (ip6_local_full_reassembly_handoff_node)
1911 (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
1912 {
1913   return ip6_full_reassembly_handoff_inline (
1914     vm, node, frame, false /* is_feature */, true /* is_feature */);
1915 }
1916
1917 VLIB_REGISTER_NODE (ip6_local_full_reassembly_handoff_node) = {
1918   .name = "ip6-local-full-reassembly-handoff",
1919   .vector_size = sizeof (u32),
1920   .n_errors = ARRAY_LEN(ip6_full_reassembly_handoff_error_strings),
1921   .error_strings = ip6_full_reassembly_handoff_error_strings,
1922   .format_trace = format_ip6_full_reassembly_handoff_trace,
1923
1924   .n_next_nodes = 1,
1925
1926   .next_nodes = {
1927     [0] = "error-drop",
1928   },
1929 };
1930
1931 VLIB_NODE_FN (ip6_full_reassembly_feature_handoff_node) (vlib_main_t * vm,
1932                                vlib_node_runtime_t * node, vlib_frame_t * frame)
1933 {
1934   return ip6_full_reassembly_handoff_inline (
1935     vm, node, frame, true /* is_feature */, false /* is_local */);
1936 }
1937
1938 VLIB_REGISTER_NODE (ip6_full_reassembly_feature_handoff_node) = {
1939   .name = "ip6-full-reass-feature-hoff",
1940   .vector_size = sizeof (u32),
1941   .n_errors = ARRAY_LEN(ip6_full_reassembly_handoff_error_strings),
1942   .error_strings = ip6_full_reassembly_handoff_error_strings,
1943   .format_trace = format_ip6_full_reassembly_handoff_trace,
1944
1945   .n_next_nodes = 1,
1946
1947   .next_nodes = {
1948     [0] = "error-drop",
1949   },
1950 };
1951
1952 #ifndef CLIB_MARCH_VARIANT
1953 int
1954 ip6_full_reass_enable_disable_with_refcnt (u32 sw_if_index, int is_enable)
1955 {
1956   ip6_full_reass_main_t *rm = &ip6_full_reass_main;
1957   vec_validate (rm->feature_use_refcount_per_intf, sw_if_index);
1958   if (is_enable)
1959     {
1960       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1961         {
1962           ++rm->feature_use_refcount_per_intf[sw_if_index];
1963           return vnet_feature_enable_disable ("ip6-unicast",
1964                                               "ip6-full-reassembly-feature",
1965                                               sw_if_index, 1, 0, 0);
1966         }
1967       ++rm->feature_use_refcount_per_intf[sw_if_index];
1968     }
1969   else
1970     {
1971       --rm->feature_use_refcount_per_intf[sw_if_index];
1972       if (!rm->feature_use_refcount_per_intf[sw_if_index])
1973         return vnet_feature_enable_disable ("ip6-unicast",
1974                                             "ip6-full-reassembly-feature",
1975                                             sw_if_index, 0, 0, 0);
1976     }
1977   return -1;
1978 }
1979
1980 void
1981 ip6_local_full_reass_enable_disable (int enable)
1982 {
1983   if (enable)
1984     {
1985       if (!ip6_full_reass_main.is_local_reass_enabled)
1986         {
1987           ip6_full_reass_main.is_local_reass_enabled = 1;
1988           ip6_register_protocol (IP_PROTOCOL_IPV6_FRAGMENTATION,
1989                                  ip6_local_full_reass_node.index);
1990         }
1991     }
1992   else
1993     {
1994       if (ip6_full_reass_main.is_local_reass_enabled)
1995         {
1996           ip6_full_reass_main.is_local_reass_enabled = 0;
1997           ip6_unregister_protocol (IP_PROTOCOL_IPV6_FRAGMENTATION);
1998         }
1999     }
2000 }
2001
2002 int
2003 ip6_local_full_reass_enabled ()
2004 {
2005   return ip6_full_reass_main.is_local_reass_enabled;
2006 }
2007
2008 #endif
2009
2010 /*
2011  * fd.io coding-style-patch-verification: ON
2012  *
2013  * Local Variables:
2014  * eval: (c-set-style "gnu")
2015  * End:
2016  */